Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(smart-crusher): preserve unchanged JSON bytes (#4044)
Mansurshakh Japarov93279f3208b1
diff --git a/crates/headroom-core/src/transforms/live_zone.rs b/crates/headroom-core/src/transforms/live_zone.rsindex a41fee3d2..72d016fe2 100644--- a/crates/headroom-core/src/transforms/live_zone.rs+++ b/crates/headroom-core/src/transforms/live_zone.rs@@ -115,6 +115,7 @@ use super::diff_compressor::{DiffCompressor, DiffCompressorConfig};#[cfg(feature = "ml")]use super::kompress::{Kompress, KompressConfig};use super::log_compressor::{LogCompressor, LogCompressorConfig};+use super::pipeline::JsonMinifier;use super::search_compressor::{SearchCompressor, SearchCompressorConfig};use super::smart_crusher::{SmartCrusher, SmartCrusherConfig};use crate::ccr::{compute_key, marker_for, CcrStore};@@ -1176,8 +1177,11 @@ fn compress_one_block(// stays honest — the marker costs ~6 tokens and we'd// rather forward the original than ship a bigger payload// for a 5-byte block.- let (compressed_for_replacement, ccr_hash_emitted) =- maybe_inject_ccr_marker(content_text, &compressed, ccr_store);+ let (compressed_for_replacement, ccr_hash_emitted) = if strategy == "json_minifier" {+ (compressed, None)+ } else {+ maybe_inject_ccr_marker(content_text, &compressed, ccr_store)+ };let compressed_bytes = compressed_for_replacement.len();// 3. Tokenizer-validated rejection. Per PR-B4 spec we// count both the original and compressed strings@@ -1659,6 +1663,12 @@ fn dispatch_compressor(text: &str, content_type: ContentType) -> DispatchResult// arrays, and returns the input.let result = smart_crusher().crush(text, EMPTY_QUERY, DEFAULT_BIAS);if !result.was_modified {+ if let Ok(Some(minified)) = JsonMinifier.minify(text) {+ return DispatchResult::Compressed {+ strategy: "json_minifier",+ compressed: minified,+ };+ }return DispatchResult::NoOp {content_type: content_type.as_str(),};diff --git a/crates/headroom-core/src/transforms/pipeline/reformats/json_minifier.rs b/crates/headroom-core/src/transforms/pipeline/reformats/json_minifier.rsindex 8989add80..1df8e61c0 100644--- a/crates/headroom-core/src/transforms/pipeline/reformats/json_minifier.rs+++ b/crates/headroom-core/src/transforms/pipeline/reformats/json_minifier.rs@@ -1,9 +1,8 @@//! `JsonMinifier` — a [`ReformatTransform`] that strips insignificant-//! whitespace from JSON via `serde_json` round-trip.+//! whitespace from validated JSON without reserializing its values.//!-//! Lossless by construction: parse to `serde_json::Value`, re-emit-//! compactly, return whichever is shorter. No CCR involvement —-//! reformat transforms never need it.+//! Strings, escapes, number literals, key order and duplicate keys remain+//! byte-identical. No CCR involvement — reformat transforms never need it.//!//! [`ReformatTransform`]: crate::transforms::pipeline::traits::ReformatTransform@@ -17,6 +16,43 @@ const NAME: &str = "json_minifier";#[derive(Debug, Default, Clone, Copy)]pub struct JsonMinifier;+impl JsonMinifier {+ /// Return a smaller data-equivalent JSON representation, or no rewrite.+ pub fn minify(&self, content: &str) -> Result<Option<String>, TransformError> {+ if content.trim().is_empty() {+ return Err(TransformError::skipped(NAME, "empty input"));+ }+ let _: &serde_json::value::RawValue = serde_json::from_str(content)+ .map_err(|e| TransformError::invalid_input(NAME, e.to_string()))?;+ let mut output: Option<String> = None;+ let mut segment_start = 0;+ let mut quoted = false;+ let mut escaped = false;+ for (index, byte) in content.bytes().enumerate() {+ if quoted {+ if escaped {+ escaped = false;+ } else if byte == b'\\' {+ escaped = true;+ } else if byte == b'"' {+ quoted = false;+ }+ } else if byte == b'"' {+ quoted = true;+ } else if matches!(byte, b' ' | b'\n' | b'\r' | b'\t') {+ output+ .get_or_insert_with(|| String::with_capacity(content.len() - 1))+ .push_str(&content[segment_start..index]);+ segment_start = index + 1;+ }+ }+ if let Some(output) = &mut output {+ output.push_str(&content[segment_start..]);+ }+ Ok(output)+ }+}+impl ReformatTransform for JsonMinifier {fn name(&self) -> &'static str {NAME@@ -30,40 +66,25 @@ impl ReformatTransform for JsonMinifier {}fn apply(&self, content: &str) -> Result<ReformatOutput, TransformError> {- let trimmed = content.trim();- if trimmed.is_empty() {- return Err(TransformError::skipped(NAME, "empty input"));- }-- let value: serde_json::Value = serde_json::from_str(trimmed)- .map_err(|e| TransformError::invalid_input(NAME, e.to_string()))?;-- let minified = serde_json::to_string(&value)- .map_err(|e| TransformError::internal(NAME, e.to_string()))?;-- // Defensive: if minification grew the byte count (e.g. caller- // already passed compact JSON, or escaping rules added bytes),- // hand the original back so we never inflate the wire output.- if minified.len() >= content.len() {- return Ok(ReformatOutput::from_lengths(- content.len(),- content.to_string(),- ));- }-- Ok(ReformatOutput::from_lengths(content.len(), minified))+ let output = self.minify(content)?.unwrap_or_else(|| content.to_string());+ Ok(ReformatOutput::from_lengths(content.len(), output))}}#[cfg(test)]mod tests {use super::*;-#[test]- fn name_and_applies_to() {- let m = JsonMinifier;- assert_eq!(m.name(), "json_minifier");- assert_eq!(m.applies_to(), &[ContentType::JsonArray]);+ fn minification_preserves_json_literals() {+ let input =+ r#" { "z": 123456789012345678901234567890, "a": 1.2300e+400, "z": "\u0061 \" b 🌍" } "#;+ let compact =+ r#"{"z":123456789012345678901234567890,"a":1.2300e+400,"z":"\u0061 \" b 🌍"}"#;+ assert_eq!(+ JsonMinifier.minify(input).unwrap().as_deref(),+ Some(compact)+ );+ assert_eq!(JsonMinifier.minify(compact).unwrap(), None);}#[test]diff --git a/crates/headroom-core/src/transforms/smart_crusher/crusher.rs b/crates/headroom-core/src/transforms/smart_crusher/crusher.rsindex ff5624760..c626ae240 100644--- a/crates/headroom-core/src/transforms/smart_crusher/crusher.rs+++ b/crates/headroom-core/src/transforms/smart_crusher/crusher.rs@@ -372,14 +372,13 @@ impl SmartCrusher {////// Parses `content` as JSON, recursively processes it (compressing/// arrays at every depth via the appropriate per-type crusher),- /// then re-serializes with Python-compatible formatting (`, ` and- /// `: ` separators, ASCII-escaped non-ASCII).+ /// then returns original bytes for unchanged values. Transformed+ /// values use compact JSON, unescaped Unicode, and insertion order.////// Returns a `CrushResult` with:- /// - `compressed`: the re-serialized JSON.+ /// - `compressed`: transformed JSON, or original bytes on passthrough./// - `original`: the input string (unmodified).- /// - `was_modified`: whether `compressed` differs from `content`'s- /// trimmed form.+ /// - `was_modified`: whether processing changed the parsed JSON value./// - `strategy`: combined strategy info from all crushed arrays/// (or `"passthrough"`).pub fn crush(&self, content: &str, query: &str, bias: f64) -> CrushResult {@@ -418,9 +417,9 @@ impl SmartCrusher {}}- /// `SmartCrusher._smart_crush_content` (Python line 2243-2301).- /// JSON-parse, recursively process, re-serialize. CCR marker- /// injection is stubbed (CCR is disabled in this stage).+ /// Apply recursive JSON transforms without rewriting unchanged values.+ /// Actual transformations use compact JSON; unchanged values retain+ /// the original bytes, including whitespace and escapes.////// Returns `(crushed_content, was_modified, info)`.pub fn smart_crush_content(@@ -446,19 +445,12 @@ impl SmartCrusher {let (crushed, info) =self.process_value_with_hook(&parsed, 0, query_context, bias, prose_hook);- // When an object and its descendants were left unchanged, keep- // the original bytes. Re-serializing would rewrite Unicode- // escapes and numeric lexical forms, then advertise a false- // compression. Nested skip/compaction still serializes below- // because those paths populate `info`. Spaced JSON without a- // nested strategy is compact-serialized for historical crush()- // / parity fixtures.- if crushed == parsed- && info.is_empty()- && matches!(parsed, Value::Object(_))- && !has_json_insignificant_whitespace(content)- {- return (content.to_string(), false, String::new());+ // An unchanged value is passthrough, regardless of container type+ // or whitespace. Preserve its bytes, escapes, and numeric forms+ // instead of advertising re-serialization as compression. Keep+ // diagnostic strategy information for decisions that made no edit.+ if crushed == parsed {+ return (content.to_string(), false, info);}// Re-serialize with Python `safe_json_dumps` formatting:@@ -1295,36 +1287,6 @@ fn hash_canonical(canonical: &str) -> String {// here because `process_string`'s `string_ccr:<kind>` strategy-info// label is local to this module's debug-string convention.-/// True when `s` has JSON whitespace outside of string literals-/// (spaces after `:` / `,`, pretty-print newlines, etc.). Used to-/// keep compact unchanged objects on the original-bytes path while-/// still compact-serializing spaced inputs for historical crush()-/// output.-fn has_json_insignificant_whitespace(s: &str) -> bool {- let mut in_string = false;- let mut escape = false;- for ch in s.trim().chars() {- if in_string {- if escape {- escape = false;- } else if ch == '\\' {- escape = true;- } else if ch == '"' {- in_string = false;- }- continue;- }- if ch == '"' {- in_string = true;- continue;- }- if ch.is_whitespace() {- return true;- }- }- false-}-fn opaque_kind_label(kind: &super::compaction::OpaqueKind) -> &str {use super::compaction::OpaqueKind;match kind {@@ -2004,36 +1966,45 @@ mod tests {}#[test]- fn crush_unchanged_object_keeps_original_bytes() {+ fn crush_unchanged_values_preserve_original_bytes() {let c = crusher();- // Compact input: escaped Unicode and numeric lexical forms must- // survive the passthrough branch rather than being reserialized- // (`\u00e9` → `é`, `1.0` → `1`).- let compact = r#"{"cafe":"caf\u00e9","n":1.0,"ok":true}"#;- let result = c.crush(compact, "", 1.0);- assert_eq!(result.compressed, compact);- assert!(!result.was_modified);- assert_eq!(result.strategy, "passthrough");- assert!(- !result.compressed.contains("<<ccr:"),- "unchanged object must not invent a CCR marker"- );+ let cases = [+ r#"{"cafe":"caf\u00e9","n":1.0,"ok":true}"#,+ r#"{ "cafe": "caf\u00e9", "n": 1.0, "ok": true }"#,+ " \n{\n \"cafe\": \"caf\\u00e9\", \"n\": 1.0\n}\n\t",+ r#"[ "\u0061", 1.0, { "ok": true } ]"#,+ " \t\"caf\\u00e9\"\r\n ",+ " \n1.0e+03\t ",+ " \tnull\r\n ",+ ];+ for input in cases {+ let result = c.crush(input, "", 1.0);+ assert_eq!(+ result.compressed, input,+ "passthrough must preserve the original representation"+ );+ assert!(!result.was_modified);+ assert_eq!(result.strategy, "passthrough");+ assert!(+ !result.compressed.contains("<<ccr:"),+ "unchanged values must not invent CCR retrieval markers"+ );+ }+ }- // Spaced / pretty-printed unchanged objects still compact-serialize- // (historical crush() output, required by parity fixtures) but- // must not advertise a key-drop strategy or invent a CCR hash.- let spaced = "{ \"cafe\": \"caf\\u00e9\", \"n\": 1.0, \"ok\": true }";- let spaced_result = c.crush(spaced, "", 1.0);- assert_eq!(spaced_result.strategy, "passthrough");- assert!(- !spaced_result.strategy.contains("adaptive"),- "must not report a key-drop strategy: {}",- spaced_result.strategy- );- assert!(!spaced_result.compressed.contains("<<ccr:"));- let parsed: Value = serde_json::from_str(&spaced_result.compressed).unwrap();- assert_eq!(parsed["cafe"], "café");- assert_eq!(parsed["ok"], true);+ #[test]+ fn crush_unchanged_analyzed_values_do_not_report_compression() {+ let c = SmartCrusher::without_compaction(SmartCrusherConfig::default());+ let rows: Vec<Value> = (0..15)+ .map(|i| json!({"i": i, "kind": "deep", "v": format!("x{i}")}))+ .collect();+ let nested = serde_json::to_string_pretty(&json!({"a": {"b": {"events": rows}}}))+ .expect("valid nested JSON");+ for input in ["[null, true, false, null, true, false, null]", &nested] {+ let result = c.crush(input, "", 1.0);+ assert_eq!(result.compressed, input);+ assert!(!result.was_modified);+ }}#[test]diff --git a/crates/headroom-core/tests/live_zone_ccr.rs b/crates/headroom-core/tests/live_zone_ccr.rsindex 7f973d2d2..c34833b48 100644--- a/crates/headroom-core/tests/live_zone_ccr.rs+++ b/crates/headroom-core/tests/live_zone_ccr.rs@@ -117,3 +117,31 @@ fn store_only_populated_after_token_gate_admits() {.expect("dispatcher");assert_eq!(store.len(), 0, "no compression → no CCR put");}++#[test]+fn lossless_json_minification_stays_valid_without_a_retrieval_obligation() {+ let fields: serde_json::Map<String, Value> =+ (0..500).map(|i| (format!("field_{i}"), json!(i))).collect();+ let value = json!([fields]);+ let payload = serde_json::to_string_pretty(&value).unwrap();+ let body = body_with_payload(&payload);+ let store = InMemoryCcrStore::new();+ let outcome = compress_anthropic_live_zone_with_ccr(+ &body,+ 0,+ AuthMode::Payg,+ DEFAULT_MODEL,+ Some(&store),+ )+ .unwrap();+ let LiveZoneOutcome::Modified { new_body, .. } = outcome else {+ panic!("pretty JSON must save tokens through the lossless minifier");+ };+ let forwarded: Value = serde_json::from_str(new_body.get()).unwrap();+ let content = forwarded["messages"][0]["content"][0]["content"]+ .as_str()+ .unwrap();+ assert_eq!(serde_json::from_str::<Value>(content).unwrap(), value);+ assert_eq!(content, serde_json::to_string(&value).unwrap());+ assert!(store.get(&compute_key(payload.as_bytes())).is_none());+}diff --git a/crates/headroom-py/src/lib.rs b/crates/headroom-py/src/lib.rsindex 52d9519a5..c56739d7e 100644--- a/crates/headroom-py/src/lib.rs+++ b/crates/headroom-py/src/lib.rs@@ -18,6 +18,7 @@ use std::collections::BTreeMap;use headroom_core::signals::{ImportanceCategory, ImportanceContext, KeywordDetector, KeywordRegistry, LineImportanceDetector,};+use headroom_core::transforms::pipeline::JsonMinifier;use headroom_core::transforms::smart_crusher::compaction::{ClassifyConfig, CompactConfig, DocumentCompactor,};@@ -47,6 +48,7 @@ use headroom_core::transforms::{TextCrusherResult as RustTextCrusherResult,};use pyo3::prelude::*;+use pyo3::pybacked::PyBackedStr;use pyo3::types::{PyBytes, PyDict};/// Identity stub used by the Python smoke test to verify linkage.@@ -55,6 +57,13 @@ fn hello() -> &'static str {headroom_core::hello()}+/// Run the explicit JSON reformat without copying the immutable Python input.+#[pyfunction]+fn minify_json(py: Python<'_>, content: PyBackedStr) -> PyResult<Option<String>> {+ py.detach(|| JsonMinifier.minify(content.as_str()))+ .map_err(|error| pyo3::exceptions::PyValueError::new_err(error.to_string()))+}+fn type_name(v: &serde_json::Value) -> &'static str {match v {serde_json::Value::Null => "null",@@ -1814,6 +1823,7 @@ fn _core(m: &Bound<'_, PyModule>) -> PyResult<()> {let _ = pyo3_log::try_init();m.add_function(wrap_pyfunction!(hello, m)?)?;+ m.add_function(wrap_pyfunction!(minify_json, m)?)?;m.add_class::<PyDiffCompressorConfig>()?;m.add_class::<PyDiffCompressionResult>()?;m.add_class::<PyDiffCompressorStats>()?;diff --git a/docs/content/docs/smart-crusher.mdx b/docs/content/docs/smart-crusher.mdxindex cd7e335f4..291866969 100644--- a/docs/content/docs/smart-crusher.mdx+++ b/docs/content/docs/smart-crusher.mdx@@ -32,6 +32,61 @@ On that path, the target size is `max_items_after_crush` (15 by default): a 1,00| Relevant | Top K | Match user's query || Others | Sampled | Statistical representation |+### JSON objects and unchanged values++Object keys are never sampled or dropped: named fields are not interchangeable+array rows. Removing a key without a retrieval marker would make its value+unrecoverable. SmartCrusher instead recurses into each value, so nested arrays+can still use lossless table compaction or CCR-backed row offload, and opaque+strings can still be replaced by retrievable CCR references.++A wide object containing only unchanged values therefore has **zero structural+compression savings**, regardless of how many keys it contains. This is a+deliberate reversibility tradeoff, not a promise that every large JSON object+will shrink. Key-level offload would need its own retrieval contract before+object-key sampling could safely return.++When recursive processing leaves a valid JSON value unchanged, SmartCrusher+returns its original bytes and sets `was_modified=False`. This applies to+objects, arrays, and scalars, including whitespace, Unicode escapes, and numeric+representations. Actual structural transformations still use compact JSON+serialization.++The routing pipeline then applies the existing Rust `JsonMinifier` as a separate+lossless step when SmartCrusher leaves routed JSON unchanged. Savings are+attributed to `json_minifier`, not `smart_crusher`, and do not create structural+TOIN records. Minification removes only whitespace outside strings: key order,+duplicate keys, string escapes, and number spellings remain byte-stable. Repeating+the same input produces the same output; already-minified JSON is a no-op.+The native live-zone JSON-array arm follows the same explicit step; its existing+content-type and token-reduction gates still apply.+Because no information is offloaded, lossless minification does not store an+original or append a CCR retrieval marker: the forwarded value remains valid+JSON. A mixed document still reports `mixed` when its sections take different+routes, even if only its JSON section saves tokens.+Minification takes precedence over lossy fallbacks when it saves tokens;+already-compact input can still reach the existing fallback chain. For tool+results with CCR enabled, each changed lossy mixed section must carry its own+retrieval marker. Embedded JSON carries reversibility separately for each adopted+span: lossless minification needs no marker, while a marker on another span or+section does not authorize dropping its data. Candidates rejected by the token+benefit gate do not affect the adopted spans' reversibility. Rejected mixed+results do not enter the tool-result cache.++### Comparing historical TOIN data++Unchanged values do not produce new TOIN compression records, even when the+strategy contains a diagnostic about a skipped transformation. Older TOIN+aggregates may include object-scoring strategies, formatting-only changes, or+zero-savings `lossless_only` decisions from different implementations.++Do not compare those aggregates directly with current object-key-preserving+results or treat zero-savings decisions as successful compression. Compare the+same corpus and configuration on both revisions, report unchanged wide objects+separately from objects with transformed descendants, and identify the engine+revision. Existing TOIN history is retained; this change does not rewrite or+delete a user's learned state.+## Quick Start<Tabs groupId="lang" items={['TypeScript', 'Python']}>diff --git a/headroom/transforms/content_router.py b/headroom/transforms/content_router.pyindex 994dd341d..3359f0c75 100644--- a/headroom/transforms/content_router.py+++ b/headroom/transforms/content_router.py@@ -2031,6 +2031,7 @@ class CompressionStrategy(Enum):CODE_AWARE = "code_aware"SMART_CRUSHER = "smart_crusher"+ JSON_MINIFIER = "json_minifier"SEARCH = "search"LOG = "log"KOMPRESS = "kompress"@@ -2117,6 +2118,8 @@ class RouterCompressionResult:LOG fallback chain it's three. Lets log readers see *how*we got to the final compressor without parsing thedecision_reason string.+ unrecoverable_sections: A changed lossy mixed section or adopted embedded+ JSON span lacks its own recovery, independently of sibling markers.cache_hit: True when this result was reused from a cacheinstead of a fresh compression run. compress() itselfnever sets this (only apply() has the router-internal@@ -2132,6 +2135,7 @@ class RouterCompressionResult:sections_processed: int = 1strategy_chain: list[str] = field(default_factory=list)cache_hit: bool = False+ unrecoverable_sections: bool = False@propertydef total_original_tokens(self) -> int:@@ -3171,8 +3175,8 @@ class ContentRouter(Transform):language: Optional language hint for code.context: Query context for pattern learning."""- # Skip SmartCrusher - it handles its own TOIN recording- if strategy == CompressionStrategy.SMART_CRUSHER:+ # SmartCrusher records its own decisions; reformatting does not train TOIN.+ if strategy in {CompressionStrategy.SMART_CRUSHER, CompressionStrategy.JSON_MINIFIER}:return# Skip if no actual compression happened@@ -3599,6 +3603,7 @@ class ContentRouter(Transform):compressed_sections: list[str] = []routing_log: list[RoutingDecision] = []+ unrecoverable_sections = Falsefor i, section in enumerate(sections):if placeholders and any(ph in section.content for ph in placeholders):@@ -3626,15 +3631,27 @@ class ContentRouter(Transform):# Compress sectionoriginal_tokens = _estimate_tokens(section.content)- compressed_content, compressed_tokens, _section_chain = self._apply_strategy_to_content(- section.content,- strategy,- context,- section.language,- question,- bias=bias,+ compressed_content, compressed_tokens, _section_chain, reversible = (+ self._apply_strategy_to_content(+ section.content,+ strategy,+ context,+ section.language,+ question,+ bias=bias,+ ))+ if _section_chain == [CompressionStrategy.JSON_MINIFIER.value]:+ strategy = CompressionStrategy.JSON_MINIFIER+ if (+ compressed_content != section.content+ and not reversible+ and (+ strategy in self.LOSSY_UNMARKED_STRATEGIES or "embedded_json" in _section_chain+ )+ ):+ unrecoverable_sections = True# Preserve code fence markersif section.is_code_fence and section.language:compressed_content = f"```{section.language}\n{compressed_content}\n```"@@ -3671,13 +3688,19 @@ class ContentRouter(Transform):compressed = "\n\n".join(compressed_sections)if protected:compressed = restore_tags(compressed, protected)+ routed_strategies = {decision.strategy for decision in routing_log}return RouterCompressionResult(compressed=compressed,original=content,- strategy_used=CompressionStrategy.MIXED,+ strategy_used=(+ next(iter(routed_strategies))+ if len(routed_strategies) == 1+ else CompressionStrategy.MIXED+ ),routing_log=routing_log,sections_processed=len(sections),+ unrecoverable_sections=unrecoverable_sections,)def _compress_pure(@@ -3702,15 +3725,18 @@ class ContentRouter(Transform):"""original_tokens = _estimate_tokens(content)- compressed, compressed_tokens, strategy_chain = self._apply_strategy_to_content(+ compressed, compressed_tokens, strategy_chain, reversible = self._apply_strategy_to_content(content, strategy, context, question=question, bias=bias)+ if strategy_chain == [CompressionStrategy.JSON_MINIFIER.value]:+ strategy = CompressionStrategy.JSON_MINIFIERreturn RouterCompressionResult(compressed=compressed,original=content,strategy_used=strategy,strategy_chain=strategy_chain,+ unrecoverable_sections=not reversible and "embedded_json" in strategy_chain,routing_log=[RoutingDecision(content_type=self._content_type_from_strategy(strategy),@@ -4252,7 +4278,7 @@ class ContentRouter(Transform):question: str | None = None,bias: float = 1.0,_allow_embedded: bool = True,- ) -> tuple[str, int, list[str]]:+ ) -> tuple[str, int, list[str], bool]:"""Apply a compression strategy to content.Args:@@ -4265,8 +4291,11 @@ class ContentRouter(Transform):Returns:Tuple of (compressed_content, compressed_token_count,- strategy_chain). The chain lists every strategy attempted- in order — first the requested one, then any fallbacks.+ strategy_chain, reversible). Reversible means unchanged, known+ lossless, or covered by this result's own retrieval marker(s).+ Embedded routing aggregates only adopted spans; a marker in one+ span cannot cover another span's unmarked loss.+ The chain lists every strategy attempted in order.Single-entry chain means a direct hit; multi-entry meansthe fallback chain fired (e.g. ``[smart_crusher, kompress,log]``). Log readers use this to see *how* we got to the@@ -4275,7 +4304,7 @@ class ContentRouter(Transform):# Git status is path/state ground truth, not a rectangular CSV table.# Preserve it before embedded-JSON, lossy-after-fold and external stages.if _try_detect_git_status(_strip_detection_envelope(content)) is not None:- return content, _estimate_tokens(content), [CompressionStrategy.TABULAR.value]+ return content, _estimate_tokens(content), [CompressionStrategy.TABULAR.value], True# ── STRUCTURAL (embedded) JSON routing ───────────────────────────────# Before anything else: if this block is not a single JSON value but@@ -4304,7 +4333,8 @@ class ContentRouter(Transform):if _allow_embedded and not _defer_embedded_for_html:routed = self._embedded_json_route(content, context, question, bias)if routed is not None:- return routed, _estimate_tokens(routed), ["embedded_json"]+ text, reversible = routed+ return text, _estimate_tokens(text), ["embedded_json"], reversible# Track original tokens for TOIN recordingoriginal_tokens = _estimate_tokens(content)@@ -4336,8 +4366,8 @@ class ContentRouter(Transform):# never a marker-free lossy drop that could not be recovered.if self.config.lossless:if _ll_label is not None:- return _ll_content, _estimate_tokens(_ll_content), [_ll_label]- return content, original_tokens, [CompressionStrategy.PASSTHROUGH.value]+ return _ll_content, _estimate_tokens(_ll_content), [_ll_label], True+ return content, original_tokens, [CompressionStrategy.PASSTHROUGH.value], True# ── LOSSY / CCR mode: layer relevance-split + lossy ON TOP of the fold ─# The operator has opted into lossy compression, so we reclaim more than@@ -4360,7 +4390,12 @@ class ContentRouter(Transform):kind = "log" if strategy is CompressionStrategy.LOG else "search"split = self._relevance_split_compress(content, kind, context)if split is not None:- return split, _estimate_tokens(split), [kind, "relevance_split"]+ return (+ split,+ _estimate_tokens(split),+ [kind, "relevance_split"],+ bool(CCR_RETRIEVAL_MARKER_RE.search(split)),+ )# No relevance split adopted → return the STAGE 0 lossless fold as the# floor. Lossless-then-lossy: before returning, run the aggressive lossy@@ -4393,8 +4428,9 @@ class ContentRouter(Transform):_komp,_komp_tokens,[_ll_label, CompressionStrategy.KOMPRESS.value],+ bool(CCR_RETRIEVAL_MARKER_RE.search(_komp)),)- return _ll_content, _estimate_tokens(_ll_content), [_ll_label]+ return _ll_content, _estimate_tokens(_ll_content), [_ll_label], True# CCR/lossy mode, nothing foldable (code/json/text/mixed) and no relevance# split → fall through to the lossy compressors below (kompress /@@ -4411,7 +4447,7 @@ class ContentRouter(Transform):# through to the EXISTING built-in dispatch UNCHANGED.external = self._try_external_compressor(content, strategy, context, question)if external is not None:- return external+ return (*external, bool(CCR_RETRIEVAL_MARKER_RE.search(external[0])))try:if strategy == CompressionStrategy.CODE_AWARE:@@ -4503,6 +4539,18 @@ class ContentRouter(Transform):compressed = output.contentcompressed_tokens = _estimate_tokens(output.content)decision_reason = "smart_crusher"+ if compressed == content:+ from headroom._core import minify_json++ minified = minify_json(content)+ if minified is not None:+ compressed = minified+ compressed_tokens = _estimate_tokens(minified)+ strategy = CompressionStrategy.JSON_MINIFIER+ actual_strategy = strategy+ strategy_chain = [strategy.value]+ compressor_name = "JsonMinifier"+ decision_reason = "json_minifier"# Fallback to Kompress (and possibly Log) is# handled by the unified post-strategy block below# — no inline fallback here to avoid duplicate@@ -4850,7 +4898,12 @@ class ContentRouter(Transform):language=language,context=context,)- return compressed, compressed_tokens, strategy_chain+ reversible = (+ compressed == content+ or strategy_chain == [CompressionStrategy.JSON_MINIFIER.value]+ or bool(CCR_RETRIEVAL_MARKER_RE.search(compressed))+ )+ return compressed, compressed_tokens, strategy_chain, reversible# #3609: deferred embedded-JSON splice. The HTML strategy ran above and# extracted nothing (``compressed`` is still None), so the local JSON@@ -4860,7 +4913,8 @@ class ContentRouter(Transform):routed = self._embedded_json_route(content, context, question, bias)if routed is not None:strategy_chain.append("embedded_json")- return routed, _estimate_tokens(routed), strategy_chain+ text, reversible = routed+ return text, _estimate_tokens(text), strategy_chain, reversible# Fallback: return unchangedif self.config.enable_dense_line_elision:@@ -4877,7 +4931,12 @@ class ContentRouter(Transform):language=language,context=context,)- return elided, elided_tokens, strategy_chain+ return (+ elided,+ elided_tokens,+ strategy_chain,+ bool(CCR_RETRIEVAL_MARKER_RE.search(elided)),+ )strategy_chain.append(CompressionStrategy.PASSTHROUGH.value)if _router_debug_enabled():_log_router_debug(@@ -4899,7 +4958,7 @@ class ContentRouter(Transform):output=content,error=error,)- return content, original_tokens, strategy_chain+ return content, original_tokens, strategy_chain, Truedef _elide_dense(self, text: str, context: str) -> tuple[str, int] | None:"""Dense-line elision with a CCR retrieval marker; None when no line is dense.@@ -4959,19 +5018,19 @@ class ContentRouter(Transform):def _embedded_json_route(self, content: str, context: str, question: str | None, bias: float- ) -> str | None:+ ) -> tuple[str, bool] | None:"""Route embedded JSON spans inside ``content`` through the JSON compressors.Extracted from the ``_apply_strategy_to_content`` pre-pass so the HTMLstrategy can defer the splice (#3609) and retry it in the passthrough- fallback. Returns the spliced block, or ``None`` when nothing- safe/smaller applied.+ fallback. Returns the spliced block and its per-span reversibility,+ or None when nothing smaller applied."""from headroom.transforms.recursive_json import route_embedded_json- def _dispatch_span(span: str) -> str | None:+ def _dispatch_span(span: str) -> tuple[str, bool] | None:strat = self._strategy_from_detection_type(_detect_content(span).content_type)- text, _t, _c = self._apply_strategy_to_content(+ text, _t, _c, reversible = self._apply_strategy_to_content(span,strat,context,@@ -4979,7 +5038,7 @@ class ContentRouter(Transform):bias=bias,_allow_embedded=False,)- return text if text != span else None+ return (text, reversible) if text != span else Nonereturn route_embedded_json(content, _dispatch_span, tok=_estimate_tokens)@@ -5286,6 +5345,7 @@ class ContentRouter(Transform):mapping = {CompressionStrategy.CODE_AWARE: ContentType.SOURCE_CODE,CompressionStrategy.SMART_CRUSHER: ContentType.JSON_ARRAY,+ CompressionStrategy.JSON_MINIFIER: ContentType.JSON_ARRAY,CompressionStrategy.SEARCH: ContentType.SEARCH_RESULTS,CompressionStrategy.LOG: ContentType.BUILD_OUTPUT,CompressionStrategy.DIFF: ContentType.GIT_DIFF,@@ -7170,8 +7230,13 @@ class ContentRouter(Transform):if (enforce_revand self.config.ccr_inject_marker- and result.strategy_used in self.LOSSY_UNMARKED_STRATEGIES- and not CCR_RETRIEVAL_MARKER_RE.search(result.compressed)+ and (+ getattr(result, "unrecoverable_sections", False)+ or (+ result.strategy_used in self.LOSSY_UNMARKED_STRATEGIES+ and not CCR_RETRIEVAL_MARKER_RE.search(result.compressed)+ )+ )):self._cache.mark_skip(content_key)result_slots[slot_idx] = message@@ -8313,8 +8378,13 @@ class ContentRouter(Transform):if (enforce_reversibilityand self.config.ccr_inject_marker- and result.strategy_used in self.LOSSY_UNMARKED_STRATEGIES- and not CCR_RETRIEVAL_MARKER_RE.search(result.compressed)+ and (+ getattr(result, "unrecoverable_sections", False)+ or (+ result.strategy_used in self.LOSSY_UNMARKED_STRATEGIES+ and not CCR_RETRIEVAL_MARKER_RE.search(result.compressed)+ )+ )):self._cache.mark_skip(content_key)if route_counts is not None:diff --git a/headroom/transforms/recursive_json.py b/headroom/transforms/recursive_json.pyindex 539536b85..76e6ef1c1 100644--- a/headroom/transforms/recursive_json.py+++ b/headroom/transforms/recursive_json.py@@ -37,9 +37,9 @@ _OPEN = "[{"_CLOSE = "]}"_PAIR = {"}": "{", "]": "["}-#: A dispatch callback: given a JSON span's text, return the compressed text-#: (which may carry CCR markers) or ``None`` to leave it unchanged.-Dispatch = Callable[[str], "str | None"]+#: Return (compressed text, reversible) or None to leave the span unchanged.+#: Reversibility must cover this span, not a marker elsewhere in the document.+Dispatch = Callable[[str], "tuple[str, bool] | None"]def _match_span(text: str, start: int) -> int | None:@@ -240,10 +240,10 @@ def route_embedded_json(dispatch: Dispatch,*,tok: Callable[[str], int] | None = None,-) -> str | None:- """Route every embedded JSON span in ``content`` through ``dispatch`` and- splice the results back in place. Returns the rewritten block, or ``None``- when nothing safe/smaller applied.+) -> tuple[str, bool] | None:+ """Splice smaller dispatch results into ``content``, preserving surrounding bytes.++ Return (rewritten block, all adopted spans reversible), or None if no win.``content`` that is itself a single JSON value is intentionally skipped — thecaller already routes pure-JSON blocks; this exists for the *embedded* case.@@ -257,17 +257,22 @@ def route_embedded_json(return Nonerepls: list[tuple[int, int, str]] = []+ reversible = Truefor a, b in spans:chunk = content[a:b]if "<<ccr:" in chunk: # R1: already compressed — never re-routecontinueif not carries_record_array(chunk):continue- out = dispatch(chunk)- if out is None or out == chunk:+ result = dispatch(chunk)+ if result is None:+ continue+ out, span_reversible = result+ if out == chunk:continueif tok(out) < tok(chunk): # benefit gate (outcome, not a threshold)repls.append((a, b, out))+ reversible = reversible and span_reversibleif not repls:return None@@ -279,4 +284,4 @@ def route_embedded_json(last = bparts.append(content[last:])new = "".join(parts)- return new if tok(new) < tok(content) else None+ return (new, reversible) if tok(new) < tok(content) else Nonediff --git a/headroom/transforms/smart_crusher.py b/headroom/transforms/smart_crusher.pyindex 2cd3db299..f1991ddb1 100644--- a/headroom/transforms/smart_crusher.py+++ b/headroom/transforms/smart_crusher.py@@ -550,11 +550,9 @@ class SmartCrusher(Transform):# Bridging the gap here keeps JSON-array compressions fueling# the learning system.#- # Filter on `was_modified AND strategy != "passthrough"`. The- # Rust crusher sometimes flips `was_modified=True` from pure- # JSON re-canonicalization (whitespace normalization) without- # actually compressing — the strategy stays `"passthrough"` in- # that case, and there's no learning value in recording it.+ # Record actual transformations with a non-passthrough strategy.+ # Rust preserves original bytes when the parsed value is unchanged,+ # including no-op decisions that retain diagnostic strategy info.if record_toin and r.was_modified and r.strategy != "passthrough":self._record_to_toin(original=content,@@ -906,9 +904,8 @@ class SmartCrusher(Transform):tool_name=tool_name,):return content, False, "passthrough"- # Same passthrough filter as `crush()` — re-canonicalization of- # JSON whitespace can flip `was_modified=True` even when the- # `info` field reports `passthrough` and no compression happened.+ # Match `crush()`: unchanged values must not train TOIN, even when+ # `info` contains a diagnostic about a skipped transformation.if was_modified and info != "passthrough":self._record_to_toin(original=content,diff --git a/tests/parity/fixtures/smart_crusher/nested_3deep_with_array_bb5036c153aa.json b/tests/parity/fixtures/smart_crusher/nested_3deep_with_array_9aeaf457a2ba.jsonsimilarity index 64%rename from tests/parity/fixtures/smart_crusher/nested_3deep_with_array_bb5036c153aa.jsonrename to tests/parity/fixtures/smart_crusher/nested_3deep_with_array_9aeaf457a2ba.jsonindex 3c0638562..b8a20a9cf 100644--- a/tests/parity/fixtures/smart_crusher/nested_3deep_with_array_bb5036c153aa.json+++ b/tests/parity/fixtures/smart_crusher/nested_3deep_with_array_9aeaf457a2ba.json@@ -1,11 +1,18 @@{"config": {+ "compaction_core_field_fraction": 0.8,+ "compaction_heterogeneous_core_ratio": 0.6,+ "compaction_max_buckets": 8,+ "compaction_max_flatten_inner_keys": 6,+ "compaction_min_buckets": 2,"dedup_identical_items": true,"enabled": true,"factor_out_constants": false,"first_fraction": 0.3,"include_summaries": false,"last_fraction": 0.15,+ "lossless_min_savings_ratio": 0.15,+ "lossless_only": false,"max_items_after_crush": 15,"min_items_to_analyze": 5,"min_tokens_to_crush": 200,@@ -21,14 +28,14 @@"content": "{\"a\": {\"b\": {\"events\": [{\"i\": 0, \"kind\": \"deep\", \"v\": \"x0\"}, {\"i\": 1, \"kind\": \"deep\", \"v\": \"x1\"}, {\"i\": 2, \"kind\": \"deep\", \"v\": \"x2\"}, {\"i\": 3, \"kind\": \"deep\", \"v\": \"x3\"}, {\"i\": 4, \"kind\": \"deep\", \"v\": \"x4\"}, {\"i\": 5, \"kind\": \"deep\", \"v\": \"x5\"}, {\"i\": 6, \"kind\": \"deep\", \"v\": \"x6\"}, {\"i\": 7, \"kind\": \"deep\", \"v\": \"x7\"}, {\"i\": 8, \"kind\": \"deep\", \"v\": \"x8\"}, {\"i\": 9, \"kind\": \"deep\", \"v\": \"x9\"}, {\"i\": 10, \"kind\": \"deep\", \"v\": \"x10\"}, {\"i\": 11, \"kind\": \"deep\", \"v\": \"x11\"}, {\"i\": 12, \"kind\": \"deep\", \"v\": \"x12\"}, {\"i\": 13, \"kind\": \"deep\", \"v\": \"x13\"}, {\"i\": 14, \"kind\": \"deep\", \"v\": \"x14\"}]}}}","query": ""},- "input_sha256": "bb5036c153aa8a9e6700b751413952c4753c328cc8bd63986f68c3a5b4fb9cb6",+ "input_sha256": "9aeaf457a2bad5b296e27c04ae7e2af9501c046063f0f3a145dec295605c41a1","label": "nested_3deep_with_array","output": {- "compressed": "{\"a\":{\"b\":{\"events\":[{\"i\":0,\"kind\":\"deep\",\"v\":\"x0\"},{\"i\":1,\"kind\":\"deep\",\"v\":\"x1\"},{\"i\":2,\"kind\":\"deep\",\"v\":\"x2\"},{\"i\":3,\"kind\":\"deep\",\"v\":\"x3\"},{\"i\":4,\"kind\":\"deep\",\"v\":\"x4\"},{\"i\":5,\"kind\":\"deep\",\"v\":\"x5\"},{\"i\":6,\"kind\":\"deep\",\"v\":\"x6\"},{\"i\":7,\"kind\":\"deep\",\"v\":\"x7\"},{\"i\":8,\"kind\":\"deep\",\"v\":\"x8\"},{\"i\":9,\"kind\":\"deep\",\"v\":\"x9\"},{\"i\":10,\"kind\":\"deep\",\"v\":\"x10\"},{\"i\":11,\"kind\":\"deep\",\"v\":\"x11\"},{\"i\":12,\"kind\":\"deep\",\"v\":\"x12\"},{\"i\":13,\"kind\":\"deep\",\"v\":\"x13\"},{\"i\":14,\"kind\":\"deep\",\"v\":\"x14\"}]}}}",+ "compressed": "{\"a\": {\"b\": {\"events\": [{\"i\": 0, \"kind\": \"deep\", \"v\": \"x0\"}, {\"i\": 1, \"kind\": \"deep\", \"v\": \"x1\"}, {\"i\": 2, \"kind\": \"deep\", \"v\": \"x2\"}, {\"i\": 3, \"kind\": \"deep\", \"v\": \"x3\"}, {\"i\": 4, \"kind\": \"deep\", \"v\": \"x4\"}, {\"i\": 5, \"kind\": \"deep\", \"v\": \"x5\"}, {\"i\": 6, \"kind\": \"deep\", \"v\": \"x6\"}, {\"i\": 7, \"kind\": \"deep\", \"v\": \"x7\"}, {\"i\": 8, \"kind\": \"deep\", \"v\": \"x8\"}, {\"i\": 9, \"kind\": \"deep\", \"v\": \"x9\"}, {\"i\": 10, \"kind\": \"deep\", \"v\": \"x10\"}, {\"i\": 11, \"kind\": \"deep\", \"v\": \"x11\"}, {\"i\": 12, \"kind\": \"deep\", \"v\": \"x12\"}, {\"i\": 13, \"kind\": \"deep\", \"v\": \"x13\"}, {\"i\": 14, \"kind\": \"deep\", \"v\": \"x14\"}]}}}","original": "{\"a\": {\"b\": {\"events\": [{\"i\": 0, \"kind\": \"deep\", \"v\": \"x0\"}, {\"i\": 1, \"kind\": \"deep\", \"v\": \"x1\"}, {\"i\": 2, \"kind\": \"deep\", \"v\": \"x2\"}, {\"i\": 3, \"kind\": \"deep\", \"v\": \"x3\"}, {\"i\": 4, \"kind\": \"deep\", \"v\": \"x4\"}, {\"i\": 5, \"kind\": \"deep\", \"v\": \"x5\"}, {\"i\": 6, \"kind\": \"deep\", \"v\": \"x6\"}, {\"i\": 7, \"kind\": \"deep\", \"v\": \"x7\"}, {\"i\": 8, \"kind\": \"deep\", \"v\": \"x8\"}, {\"i\": 9, \"kind\": \"deep\", \"v\": \"x9\"}, {\"i\": 10, \"kind\": \"deep\", \"v\": \"x10\"}, {\"i\": 11, \"kind\": \"deep\", \"v\": \"x11\"}, {\"i\": 12, \"kind\": \"deep\", \"v\": \"x12\"}, {\"i\": 13, \"kind\": \"deep\", \"v\": \"x13\"}, {\"i\": 14, \"kind\": \"deep\", \"v\": \"x14\"}]}}}","strategy": "none:adaptive_at_limit(15->15)",- "was_modified": true+ "was_modified": false},- "recorded_at": "2026-04-27T06:59:41.857099+00:00",+ "recorded_at": "2026-10-08T19:16:38.005664+00:00","transform": "smart_crusher"}diff --git a/tests/parity/fixtures/smart_crusher/nested_object_with_array_b66d43299ea1.json b/tests/parity/fixtures/smart_crusher/nested_object_with_array_e257ab3bfe9a.jsonsimilarity index 65%rename from tests/parity/fixtures/smart_crusher/nested_object_with_array_b66d43299ea1.jsonrename to tests/parity/fixtures/smart_crusher/nested_object_with_array_e257ab3bfe9a.jsonindex 3c5c22c7d..afb2f9353 100644--- a/tests/parity/fixtures/smart_crusher/nested_object_with_array_b66d43299ea1.json+++ b/tests/parity/fixtures/smart_crusher/nested_object_with_array_e257ab3bfe9a.json@@ -1,11 +1,18 @@{"config": {+ "compaction_core_field_fraction": 0.8,+ "compaction_heterogeneous_core_ratio": 0.6,+ "compaction_max_buckets": 8,+ "compaction_max_flatten_inner_keys": 6,+ "compaction_min_buckets": 2,"dedup_identical_items": true,"enabled": true,"factor_out_constants": false,"first_fraction": 0.3,"include_summaries": false,"last_fraction": 0.15,+ "lossless_min_savings_ratio": 0.15,+ "lossless_only": false,"max_items_after_crush": 15,"min_items_to_analyze": 5,"min_tokens_to_crush": 200,@@ -21,14 +28,14 @@"content": "{\"request_id\": \"req-1\", \"events\": [{\"step\": 0, \"kind\": \"trace\", \"msg\": \"e0\"}, {\"step\": 1, \"kind\": \"trace\", \"msg\": \"e1\"}, {\"step\": 2, \"kind\": \"trace\", \"msg\": \"e2\"}, {\"step\": 3, \"kind\": \"trace\", \"msg\": \"e3\"}, {\"step\": 4, \"kind\": \"trace\", \"msg\": \"e4\"}, {\"step\": 5, \"kind\": \"trace\", \"msg\": \"e5\"}, {\"step\": 6, \"kind\": \"trace\", \"msg\": \"e6\"}, {\"step\": 7, \"kind\": \"trace\", \"msg\": \"e7\"}, {\"step\": 8, \"kind\": \"trace\", \"msg\": \"e8\"}, {\"step\": 9, \"kind\": \"trace\", \"msg\": \"e9\"}, {\"step\": 10, \"kind\": \"trace\", \"msg\": \"e10\"}, {\"step\": 11, \"kind\": \"trace\", \"msg\": \"e11\"}, {\"step\": 12, \"kind\": \"trace\", \"msg\": \"e12\"}, {\"step\": 13, \"kind\": \"trace\", \"msg\": \"e13\"}, {\"step\": 14, \"kind\": \"trace\", \"msg\": \"e14\"}, {\"step\": 15, \"kind\": \"trace\", \"msg\": \"e15\"}, {\"step\": 16, \"kind\": \"trace\", \"msg\": \"e16\"}, {\"step\": 17, \"kind\": \"trace\", \"msg\": \"e17\"}, {\"step\": 18, \"kind\": \"trace\", \"msg\": \"e18\"}, {\"step\": 19, \"kind\": \"trace\", \"msg\": \"e19\"}]}","query": ""},- "input_sha256": "b66d43299ea15229c3fea6607b8fc697e32e72811ce4364c71bc9f64357208f1",+ "input_sha256": "e257ab3bfe9a8a599e4516598d73de4c687f2afaf99e0fa5deca3eb3808d7841","label": "nested_object_with_array","output": {- "compressed": "{\"request_id\":\"req-1\",\"events\":[{\"step\":0,\"kind\":\"trace\",\"msg\":\"e0\"},{\"step\":1,\"kind\":\"trace\",\"msg\":\"e1\"},{\"step\":2,\"kind\":\"trace\",\"msg\":\"e2\"},{\"step\":3,\"kind\":\"trace\",\"msg\":\"e3\"},{\"step\":4,\"kind\":\"trace\",\"msg\":\"e4\"},{\"step\":5,\"kind\":\"trace\",\"msg\":\"e5\"},{\"step\":6,\"kind\":\"trace\",\"msg\":\"e6\"},{\"step\":7,\"kind\":\"trace\",\"msg\":\"e7\"},{\"step\":8,\"kind\":\"trace\",\"msg\":\"e8\"},{\"step\":9,\"kind\":\"trace\",\"msg\":\"e9\"},{\"step\":10,\"kind\":\"trace\",\"msg\":\"e10\"},{\"step\":11,\"kind\":\"trace\",\"msg\":\"e11\"},{\"step\":12,\"kind\":\"trace\",\"msg\":\"e12\"},{\"step\":13,\"kind\":\"trace\",\"msg\":\"e13\"},{\"step\":14,\"kind\":\"trace\",\"msg\":\"e14\"},{\"step\":15,\"kind\":\"trace\",\"msg\":\"e15\"},{\"step\":16,\"kind\":\"trace\",\"msg\":\"e16\"},{\"step\":17,\"kind\":\"trace\",\"msg\":\"e17\"},{\"step\":18,\"kind\":\"trace\",\"msg\":\"e18\"},{\"step\":19,\"kind\":\"trace\",\"msg\":\"e19\"}]}",+ "compressed": "{\"request_id\": \"req-1\", \"events\": [{\"step\": 0, \"kind\": \"trace\", \"msg\": \"e0\"}, {\"step\": 1, \"kind\": \"trace\", \"msg\": \"e1\"}, {\"step\": 2, \"kind\": \"trace\", \"msg\": \"e2\"}, {\"step\": 3, \"kind\": \"trace\", \"msg\": \"e3\"}, {\"step\": 4, \"kind\": \"trace\", \"msg\": \"e4\"}, {\"step\": 5, \"kind\": \"trace\", \"msg\": \"e5\"}, {\"step\": 6, \"kind\": \"trace\", \"msg\": \"e6\"}, {\"step\": 7, \"kind\": \"trace\", \"msg\": \"e7\"}, {\"step\": 8, \"kind\": \"trace\", \"msg\": \"e8\"}, {\"step\": 9, \"kind\": \"trace\", \"msg\": \"e9\"}, {\"step\": 10, \"kind\": \"trace\", \"msg\": \"e10\"}, {\"step\": 11, \"kind\": \"trace\", \"msg\": \"e11\"}, {\"step\": 12, \"kind\": \"trace\", \"msg\": \"e12\"}, {\"step\": 13, \"kind\": \"trace\", \"msg\": \"e13\"}, {\"step\": 14, \"kind\": \"trace\", \"msg\": \"e14\"}, {\"step\": 15, \"kind\": \"trace\", \"msg\": \"e15\"}, {\"step\": 16, \"kind\": \"trace\", \"msg\": \"e16\"}, {\"step\": 17, \"kind\": \"trace\", \"msg\": \"e17\"}, {\"step\": 18, \"kind\": \"trace\", \"msg\": \"e18\"}, {\"step\": 19, \"kind\": \"trace\", \"msg\": \"e19\"}]}","original": "{\"request_id\": \"req-1\", \"events\": [{\"step\": 0, \"kind\": \"trace\", \"msg\": \"e0\"}, {\"step\": 1, \"kind\": \"trace\", \"msg\": \"e1\"}, {\"step\": 2, \"kind\": \"trace\", \"msg\": \"e2\"}, {\"step\": 3, \"kind\": \"trace\", \"msg\": \"e3\"}, {\"step\": 4, \"kind\": \"trace\", \"msg\": \"e4\"}, {\"step\": 5, \"kind\": \"trace\", \"msg\": \"e5\"}, {\"step\": 6, \"kind\": \"trace\", \"msg\": \"e6\"}, {\"step\": 7, \"kind\": \"trace\", \"msg\": \"e7\"}, {\"step\": 8, \"kind\": \"trace\", \"msg\": \"e8\"}, {\"step\": 9, \"kind\": \"trace\", \"msg\": \"e9\"}, {\"step\": 10, \"kind\": \"trace\", \"msg\": \"e10\"}, {\"step\": 11, \"kind\": \"trace\", \"msg\": \"e11\"}, {\"step\": 12, \"kind\": \"trace\", \"msg\": \"e12\"}, {\"step\": 13, \"kind\": \"trace\", \"msg\": \"e13\"}, {\"step\": 14, \"kind\": \"trace\", \"msg\": \"e14\"}, {\"step\": 15, \"kind\": \"trace\", \"msg\": \"e15\"}, {\"step\": 16, \"kind\": \"trace\", \"msg\": \"e16\"}, {\"step\": 17, \"kind\": \"trace\", \"msg\": \"e17\"}, {\"step\": 18, \"kind\": \"trace\", \"msg\": \"e18\"}, {\"step\": 19, \"kind\": \"trace\", \"msg\": \"e19\"}]}","strategy": "skip:unique_entities_no_signal(20->20)",- "was_modified": true+ "was_modified": false},- "recorded_at": "2026-04-27T06:59:39.935923+00:00",+ "recorded_at": "2026-10-08T19:16:37.831265+00:00","transform": "smart_crusher"}diff --git a/tests/parity/fixtures/smart_crusher/nulls_and_bools_06d6fd6adcab.json b/tests/parity/fixtures/smart_crusher/nulls_and_bools_9f40c7fded19.jsonsimilarity index 61%rename from tests/parity/fixtures/smart_crusher/nulls_and_bools_06d6fd6adcab.jsonrename to tests/parity/fixtures/smart_crusher/nulls_and_bools_9f40c7fded19.jsonindex 041adc374..a1b39b8b3 100644--- a/tests/parity/fixtures/smart_crusher/nulls_and_bools_06d6fd6adcab.json+++ b/tests/parity/fixtures/smart_crusher/nulls_and_bools_9f40c7fded19.json@@ -1,11 +1,18 @@{"config": {+ "compaction_core_field_fraction": 0.8,+ "compaction_heterogeneous_core_ratio": 0.6,+ "compaction_max_buckets": 8,+ "compaction_max_flatten_inner_keys": 6,+ "compaction_min_buckets": 2,"dedup_identical_items": true,"enabled": true,"factor_out_constants": false,"first_fraction": 0.3,"include_summaries": false,"last_fraction": 0.15,+ "lossless_min_savings_ratio": 0.15,+ "lossless_only": false,"max_items_after_crush": 15,"min_items_to_analyze": 5,"min_tokens_to_crush": 200,@@ -21,14 +28,14 @@"content": "[null, true, false, null, true, false, null]","query": ""},- "input_sha256": "06d6fd6adcab251afc35de66e5cd54cfc21f15175bb082286cd1255b0f5bd965",+ "input_sha256": "9f40c7fded195680cbe1281d4061246c142926ca3db3c703654a9a7182591c92","label": "nulls_and_bools","output": {- "compressed": "[null,true,false,null,true,false,null]",+ "compressed": "[null, true, false, null, true, false, null]","original": "[null, true, false, null, true, false, null]","strategy": "mixed:passthrough(7->7)",- "was_modified": true+ "was_modified": false},- "recorded_at": "2026-04-27T06:59:40.029745+00:00",+ "recorded_at": "2026-10-09T03:59:17.344613+00:00","transform": "smart_crusher"}diff --git a/tests/parity/fixtures/smart_crusher/short_array_passthrough_aeb16224b61f.json b/tests/parity/fixtures/smart_crusher/short_array_passthrough_70abace7c488.jsonsimilarity index 60%rename from tests/parity/fixtures/smart_crusher/short_array_passthrough_aeb16224b61f.jsonrename to tests/parity/fixtures/smart_crusher/short_array_passthrough_70abace7c488.jsonindex fb2fc97d5..3cc405de0 100644--- a/tests/parity/fixtures/smart_crusher/short_array_passthrough_aeb16224b61f.json+++ b/tests/parity/fixtures/smart_crusher/short_array_passthrough_70abace7c488.json@@ -1,11 +1,18 @@{"config": {+ "compaction_core_field_fraction": 0.8,+ "compaction_heterogeneous_core_ratio": 0.6,+ "compaction_max_buckets": 8,+ "compaction_max_flatten_inner_keys": 6,+ "compaction_min_buckets": 2,"dedup_identical_items": true,"enabled": true,"factor_out_constants": false,"first_fraction": 0.3,"include_summaries": false,"last_fraction": 0.15,+ "lossless_min_savings_ratio": 0.15,+ "lossless_only": false,"max_items_after_crush": 15,"min_items_to_analyze": 5,"min_tokens_to_crush": 200,@@ -21,14 +28,14 @@"content": "[1, 2, 3]","query": ""},- "input_sha256": "aeb16224b61f335e6813afa323dc5d22c23b357fe6eb10ee2f8cfb3de6bfe4cf",+ "input_sha256": "70abace7c488b4acb90c60cef985af22980bdd4d58dfb275140cc909c8a3d97f","label": "short_array_passthrough","output": {- "compressed": "[1,2,3]",+ "compressed": "[1, 2, 3]","original": "[1, 2, 3]","strategy": "passthrough",- "was_modified": true+ "was_modified": false},- "recorded_at": "2026-04-27T06:59:39.895735+00:00",+ "recorded_at": "2026-10-09T03:59:17.333122+00:00","transform": "smart_crusher"}diff --git a/tests/parity/fixtures/smart_crusher/small_object_passthrough_801afa36a32c.json b/tests/parity/fixtures/smart_crusher/small_object_passthrough_7249ffeaddf5.jsonsimilarity index 61%rename from tests/parity/fixtures/smart_crusher/small_object_passthrough_801afa36a32c.jsonrename to tests/parity/fixtures/smart_crusher/small_object_passthrough_7249ffeaddf5.jsonindex c5c6e03b3..d30c58303 100644--- a/tests/parity/fixtures/smart_crusher/small_object_passthrough_801afa36a32c.json+++ b/tests/parity/fixtures/smart_crusher/small_object_passthrough_7249ffeaddf5.json@@ -1,11 +1,18 @@{"config": {+ "compaction_core_field_fraction": 0.8,+ "compaction_heterogeneous_core_ratio": 0.6,+ "compaction_max_buckets": 8,+ "compaction_max_flatten_inner_keys": 6,+ "compaction_min_buckets": 2,"dedup_identical_items": true,"enabled": true,"factor_out_constants": false,"first_fraction": 0.3,"include_summaries": false,"last_fraction": 0.15,+ "lossless_min_savings_ratio": 0.15,+ "lossless_only": false,"max_items_after_crush": 15,"min_items_to_analyze": 5,"min_tokens_to_crush": 200,@@ -21,14 +28,14 @@"content": "{\"a\": 1, \"b\": 2, \"c\": \"hello\"}","query": ""},- "input_sha256": "801afa36a32c7ed7066e80c8a28fefedbb979918e679b5640803fb5d06ef6fbf",+ "input_sha256": "7249ffeaddf54726945ac4baf298cd33fb3bfb89f672983749065d7dfdf61553","label": "small_object_passthrough","output": {- "compressed": "{\"a\":1,\"b\":2,\"c\":\"hello\"}",+ "compressed": "{\"a\": 1, \"b\": 2, \"c\": \"hello\"}","original": "{\"a\": 1, \"b\": 2, \"c\": \"hello\"}","strategy": "passthrough",- "was_modified": true+ "was_modified": false},- "recorded_at": "2026-04-27T06:59:39.895485+00:00",+ "recorded_at": "2026-10-09T03:59:17.315451+00:00","transform": "smart_crusher"}diff --git a/tests/parity/fixtures/smart_crusher/time_series_50_25cd28df5a50.json b/tests/parity/fixtures/smart_crusher/time_series_50_2c51eb9374ce.jsonsimilarity index 66%rename from tests/parity/fixtures/smart_crusher/time_series_50_25cd28df5a50.jsonrename to tests/parity/fixtures/smart_crusher/time_series_50_2c51eb9374ce.jsonindex 05622ecde..14ff2d401 100644--- a/tests/parity/fixtures/smart_crusher/time_series_50_25cd28df5a50.json+++ b/tests/parity/fixtures/smart_crusher/time_series_50_2c51eb9374ce.json@@ -1,11 +1,18 @@{"config": {+ "compaction_core_field_fraction": 0.8,+ "compaction_heterogeneous_core_ratio": 0.6,+ "compaction_max_buckets": 8,+ "compaction_max_flatten_inner_keys": 6,+ "compaction_min_buckets": 2,"dedup_identical_items": true,"enabled": true,"factor_out_constants": false,"first_fraction": 0.3,"include_summaries": false,"last_fraction": 0.15,+ "lossless_min_savings_ratio": 0.15,+ "lossless_only": false,"max_items_after_crush": 15,"min_items_to_analyze": 5,"min_tokens_to_crush": 200,@@ -21,14 +28,14 @@"content": "[{\"ts\": 1000, \"metric\": 0.0, \"host\": \"host-0\"}, {\"ts\": 1001, \"metric\": 1.5, \"host\": \"host-1\"}, {\"ts\": 1002, \"metric\": 3.0, \"host\": \"host-2\"}, {\"ts\": 1003, \"metric\": 4.5, \"host\": \"host-0\"}, {\"ts\": 1004, \"metric\": 6.0, \"host\": \"host-1\"}, {\"ts\": 1005, \"metric\": 7.5, \"host\": \"host-2\"}, {\"ts\": 1006, \"metric\": 9.0, \"host\": \"host-0\"}, {\"ts\": 1007, \"metric\": 10.5, \"host\": \"host-1\"}, {\"ts\": 1008, \"metric\": 12.0, \"host\": \"host-2\"}, {\"ts\": 1009, \"metric\": 13.5, \"host\": \"host-0\"}, {\"ts\": 1010, \"metric\": 15.0, \"host\": \"host-1\"}, {\"ts\": 1011, \"metric\": 16.5, \"host\": \"host-2\"}, {\"ts\": 1012, \"metric\": 18.0, \"host\": \"host-0\"}, {\"ts\": 1013, \"metric\": 19.5, \"host\": \"host-1\"}, {\"ts\": 1014, \"metric\": 21.0, \"host\": \"host-2\"}, {\"ts\": 1015, \"metric\": 22.5, \"host\": \"host-0\"}, {\"ts\": 1016, \"metric\": 24.0, \"host\": \"host-1\"}, {\"ts\": 1017, \"metric\": 25.5, \"host\": \"host-2\"}, {\"ts\": 1018, \"metric\": 27.0, \"host\": \"host-0\"}, {\"ts\": 1019, \"metric\": 28.5, \"host\": \"host-1\"}, {\"ts\": 1020, \"metric\": 30.0, \"host\": \"host-2\"}, {\"ts\": 1021, \"metric\": 31.5, \"host\": \"host-0\"}, {\"ts\": 1022, \"metric\": 33.0, \"host\": \"host-1\"}, {\"ts\": 1023, \"metric\": 34.5, \"host\": \"host-2\"}, {\"ts\": 1024, \"metric\": 36.0, \"host\": \"host-0\"}, {\"ts\": 1025, \"metric\": 37.5, \"host\": \"host-1\"}, {\"ts\": 1026, \"metric\": 39.0, \"host\": \"host-2\"}, {\"ts\": 1027, \"metric\": 40.5, \"host\": \"host-0\"}, {\"ts\": 1028, \"metric\": 42.0, \"host\": \"host-1\"}, {\"ts\": 1029, \"metric\": 43.5, \"host\": \"host-2\"}, {\"ts\": 1030, \"metric\": 45.0, \"host\": \"host-0\"}, {\"ts\": 1031, \"metric\": 46.5, \"host\": \"host-1\"}, {\"ts\": 1032, \"metric\": 48.0, \"host\": \"host-2\"}, {\"ts\": 1033, \"metric\": 49.5, \"host\": \"host-0\"}, {\"ts\": 1034, \"metric\": 51.0, \"host\": \"host-1\"}, {\"ts\": 1035, \"metric\": 52.5, \"host\": \"host-2\"}, {\"ts\": 1036, \"metric\": 54.0, \"host\": \"host-0\"}, {\"ts\": 1037, \"metric\": 55.5, \"host\": \"host-1\"}, {\"ts\": 1038, \"metric\": 57.0, \"host\": \"host-2\"}, {\"ts\": 1039, \"metric\": 58.5, \"host\": \"host-0\"}, {\"ts\": 1040, \"metric\": 60.0, \"host\": \"host-1\"}, {\"ts\": 1041, \"metric\": 61.5, \"host\": \"host-2\"}, {\"ts\": 1042, \"metric\": 63.0, \"host\": \"host-0\"}, {\"ts\": 1043, \"metric\": 64.5, \"host\": \"host-1\"}, {\"ts\": 1044, \"metric\": 66.0, \"host\": \"host-2\"}, {\"ts\": 1045, \"metric\": 67.5, \"host\": \"host-0\"}, {\"ts\": 1046, \"metric\": 69.0, \"host\": \"host-1\"}, {\"ts\": 1047, \"metric\": 70.5, \"host\": \"host-2\"}, {\"ts\": 1048, \"metric\": 72.0, \"host\": \"host-0\"}, {\"ts\": 1049, \"metric\": 73.5, \"host\": \"host-1\"}]","query": ""},- "input_sha256": "25cd28df5a50521ac1f669482e1fe30bf2422f555ddd1a4d7506ca686dbd0734",+ "input_sha256": "2c51eb9374cee91458257eddb1185f23ff9890cfe8c21bb07ba88d6ba1ae0d00","label": "time_series_50","output": {- "compressed": "[{\"ts\":1000,\"metric\":0.0,\"host\":\"host-0\"},{\"ts\":1001,\"metric\":1.5,\"host\":\"host-1\"},{\"ts\":1002,\"metric\":3.0,\"host\":\"host-2\"},{\"ts\":1003,\"metric\":4.5,\"host\":\"host-0\"},{\"ts\":1004,\"metric\":6.0,\"host\":\"host-1\"},{\"ts\":1005,\"metric\":7.5,\"host\":\"host-2\"},{\"ts\":1006,\"metric\":9.0,\"host\":\"host-0\"},{\"ts\":1007,\"metric\":10.5,\"host\":\"host-1\"},{\"ts\":1008,\"metric\":12.0,\"host\":\"host-2\"},{\"ts\":1009,\"metric\":13.5,\"host\":\"host-0\"},{\"ts\":1010,\"metric\":15.0,\"host\":\"host-1\"},{\"ts\":1011,\"metric\":16.5,\"host\":\"host-2\"},{\"ts\":1012,\"metric\":18.0,\"host\":\"host-0\"},{\"ts\":1013,\"metric\":19.5,\"host\":\"host-1\"},{\"ts\":1014,\"metric\":21.0,\"host\":\"host-2\"},{\"ts\":1015,\"metric\":22.5,\"host\":\"host-0\"},{\"ts\":1016,\"metric\":24.0,\"host\":\"host-1\"},{\"ts\":1017,\"metric\":25.5,\"host\":\"host-2\"},{\"ts\":1018,\"metric\":27.0,\"host\":\"host-0\"},{\"ts\":1019,\"metric\":28.5,\"host\":\"host-1\"},{\"ts\":1020,\"metric\":30.0,\"host\":\"host-2\"},{\"ts\":1021,\"metric\":31.5,\"host\":\"host-0\"},{\"ts\":1022,\"metric\":33.0,\"host\":\"host-1\"},{\"ts\":1023,\"metric\":34.5,\"host\":\"host-2\"},{\"ts\":1024,\"metric\":36.0,\"host\":\"host-0\"},{\"ts\":1025,\"metric\":37.5,\"host\":\"host-1\"},{\"ts\":1026,\"metric\":39.0,\"host\":\"host-2\"},{\"ts\":1027,\"metric\":40.5,\"host\":\"host-0\"},{\"ts\":1028,\"metric\":42.0,\"host\":\"host-1\"},{\"ts\":1029,\"metric\":43.5,\"host\":\"host-2\"},{\"ts\":1030,\"metric\":45.0,\"host\":\"host-0\"},{\"ts\":1031,\"metric\":46.5,\"host\":\"host-1\"},{\"ts\":1032,\"metric\":48.0,\"host\":\"host-2\"},{\"ts\":1033,\"metric\":49.5,\"host\":\"host-0\"},{\"ts\":1034,\"metric\":51.0,\"host\":\"host-1\"},{\"ts\":1035,\"metric\":52.5,\"host\":\"host-2\"},{\"ts\":1036,\"metric\":54.0,\"host\":\"host-0\"},{\"ts\":1037,\"metric\":55.5,\"host\":\"host-1\"},{\"ts\":1038,\"metric\":57.0,\"host\":\"host-2\"},{\"ts\":1039,\"metric\":58.5,\"host\":\"host-0\"},{\"ts\":1040,\"metric\":60.0,\"host\":\"host-1\"},{\"ts\":1041,\"metric\":61.5,\"host\":\"host-2\"},{\"ts\":1042,\"metric\":63.0,\"host\":\"host-0\"},{\"ts\":1043,\"metric\":64.5,\"host\":\"host-1\"},{\"ts\":1044,\"metric\":66.0,\"host\":\"host-2\"},{\"ts\":1045,\"metric\":67.5,\"host\":\"host-0\"},{\"ts\":1046,\"metric\":69.0,\"host\":\"host-1\"},{\"ts\":1047,\"metric\":70.5,\"host\":\"host-2\"},{\"ts\":1048,\"metric\":72.0,\"host\":\"host-0\"},{\"ts\":1049,\"metric\":73.5,\"host\":\"host-1\"}]",+ "compressed": "[{\"ts\": 1000, \"metric\": 0.0, \"host\": \"host-0\"}, {\"ts\": 1001, \"metric\": 1.5, \"host\": \"host-1\"}, {\"ts\": 1002, \"metric\": 3.0, \"host\": \"host-2\"}, {\"ts\": 1003, \"metric\": 4.5, \"host\": \"host-0\"}, {\"ts\": 1004, \"metric\": 6.0, \"host\": \"host-1\"}, {\"ts\": 1005, \"metric\": 7.5, \"host\": \"host-2\"}, {\"ts\": 1006, \"metric\": 9.0, \"host\": \"host-0\"}, {\"ts\": 1007, \"metric\": 10.5, \"host\": \"host-1\"}, {\"ts\": 1008, \"metric\": 12.0, \"host\": \"host-2\"}, {\"ts\": 1009, \"metric\": 13.5, \"host\": \"host-0\"}, {\"ts\": 1010, \"metric\": 15.0, \"host\": \"host-1\"}, {\"ts\": 1011, \"metric\": 16.5, \"host\": \"host-2\"}, {\"ts\": 1012, \"metric\": 18.0, \"host\": \"host-0\"}, {\"ts\": 1013, \"metric\": 19.5, \"host\": \"host-1\"}, {\"ts\": 1014, \"metric\": 21.0, \"host\": \"host-2\"}, {\"ts\": 1015, \"metric\": 22.5, \"host\": \"host-0\"}, {\"ts\": 1016, \"metric\": 24.0, \"host\": \"host-1\"}, {\"ts\": 1017, \"metric\": 25.5, \"host\": \"host-2\"}, {\"ts\": 1018, \"metric\": 27.0, \"host\": \"host-0\"}, {\"ts\": 1019, \"metric\": 28.5, \"host\": \"host-1\"}, {\"ts\": 1020, \"metric\": 30.0, \"host\": \"host-2\"}, {\"ts\": 1021, \"metric\": 31.5, \"host\": \"host-0\"}, {\"ts\": 1022, \"metric\": 33.0, \"host\": \"host-1\"}, {\"ts\": 1023, \"metric\": 34.5, \"host\": \"host-2\"}, {\"ts\": 1024, \"metric\": 36.0, \"host\": \"host-0\"}, {\"ts\": 1025, \"metric\": 37.5, \"host\": \"host-1\"}, {\"ts\": 1026, \"metric\": 39.0, \"host\": \"host-2\"}, {\"ts\": 1027, \"metric\": 40.5, \"host\": \"host-0\"}, {\"ts\": 1028, \"metric\": 42.0, \"host\": \"host-1\"}, {\"ts\": 1029, \"metric\": 43.5, \"host\": \"host-2\"}, {\"ts\": 1030, \"metric\": 45.0, \"host\": \"host-0\"}, {\"ts\": 1031, \"metric\": 46.5, \"host\": \"host-1\"}, {\"ts\": 1032, \"metric\": 48.0, \"host\": \"host-2\"}, {\"ts\": 1033, \"metric\": 49.5, \"host\": \"host-0\"}, {\"ts\": 1034, \"metric\": 51.0, \"host\": \"host-1\"}, {\"ts\": 1035, \"metric\": 52.5, \"host\": \"host-2\"}, {\"ts\": 1036, \"metric\": 54.0, \"host\": \"host-0\"}, {\"ts\": 1037, \"metric\": 55.5, \"host\": \"host-1\"}, {\"ts\": 1038, \"metric\": 57.0, \"host\": \"host-2\"}, {\"ts\": 1039, \"metric\": 58.5, \"host\": \"host-0\"}, {\"ts\": 1040, \"metric\": 60.0, \"host\": \"host-1\"}, {\"ts\": 1041, \"metric\": 61.5, \"host\": \"host-2\"}, {\"ts\": 1042, \"metric\": 63.0, \"host\": \"host-0\"}, {\"ts\": 1043, \"metric\": 64.5, \"host\": \"host-1\"}, {\"ts\": 1044, \"metric\": 66.0, \"host\": \"host-2\"}, {\"ts\": 1045, \"metric\": 67.5, \"host\": \"host-0\"}, {\"ts\": 1046, \"metric\": 69.0, \"host\": \"host-1\"}, {\"ts\": 1047, \"metric\": 70.5, \"host\": \"host-2\"}, {\"ts\": 1048, \"metric\": 72.0, \"host\": \"host-0\"}, {\"ts\": 1049, \"metric\": 73.5, \"host\": \"host-1\"}]","original": "[{\"ts\": 1000, \"metric\": 0.0, \"host\": \"host-0\"}, {\"ts\": 1001, \"metric\": 1.5, \"host\": \"host-1\"}, {\"ts\": 1002, \"metric\": 3.0, \"host\": \"host-2\"}, {\"ts\": 1003, \"metric\": 4.5, \"host\": \"host-0\"}, {\"ts\": 1004, \"metric\": 6.0, \"host\": \"host-1\"}, {\"ts\": 1005, \"metric\": 7.5, \"host\": \"host-2\"}, {\"ts\": 1006, \"metric\": 9.0, \"host\": \"host-0\"}, {\"ts\": 1007, \"metric\": 10.5, \"host\": \"host-1\"}, {\"ts\": 1008, \"metric\": 12.0, \"host\": \"host-2\"}, {\"ts\": 1009, \"metric\": 13.5, \"host\": \"host-0\"}, {\"ts\": 1010, \"metric\": 15.0, \"host\": \"host-1\"}, {\"ts\": 1011, \"metric\": 16.5, \"host\": \"host-2\"}, {\"ts\": 1012, \"metric\": 18.0, \"host\": \"host-0\"}, {\"ts\": 1013, \"metric\": 19.5, \"host\": \"host-1\"}, {\"ts\": 1014, \"metric\": 21.0, \"host\": \"host-2\"}, {\"ts\": 1015, \"metric\": 22.5, \"host\": \"host-0\"}, {\"ts\": 1016, \"metric\": 24.0, \"host\": \"host-1\"}, {\"ts\": 1017, \"metric\": 25.5, \"host\": \"host-2\"}, {\"ts\": 1018, \"metric\": 27.0, \"host\": \"host-0\"}, {\"ts\": 1019, \"metric\": 28.5, \"host\": \"host-1\"}, {\"ts\": 1020, \"metric\": 30.0, \"host\": \"host-2\"}, {\"ts\": 1021, \"metric\": 31.5, \"host\": \"host-0\"}, {\"ts\": 1022, \"metric\": 33.0, \"host\": \"host-1\"}, {\"ts\": 1023, \"metric\": 34.5, \"host\": \"host-2\"}, {\"ts\": 1024, \"metric\": 36.0, \"host\": \"host-0\"}, {\"ts\": 1025, \"metric\": 37.5, \"host\": \"host-1\"}, {\"ts\": 1026, \"metric\": 39.0, \"host\": \"host-2\"}, {\"ts\": 1027, \"metric\": 40.5, \"host\": \"host-0\"}, {\"ts\": 1028, \"metric\": 42.0, \"host\": \"host-1\"}, {\"ts\": 1029, \"metric\": 43.5, \"host\": \"host-2\"}, {\"ts\": 1030, \"metric\": 45.0, \"host\": \"host-0\"}, {\"ts\": 1031, \"metric\": 46.5, \"host\": \"host-1\"}, {\"ts\": 1032, \"metric\": 48.0, \"host\": \"host-2\"}, {\"ts\": 1033, \"metric\": 49.5, \"host\": \"host-0\"}, {\"ts\": 1034, \"metric\": 51.0, \"host\": \"host-1\"}, {\"ts\": 1035, \"metric\": 52.5, \"host\": \"host-2\"}, {\"ts\": 1036, \"metric\": 54.0, \"host\": \"host-0\"}, {\"ts\": 1037, \"metric\": 55.5, \"host\": \"host-1\"}, {\"ts\": 1038, \"metric\": 57.0, \"host\": \"host-2\"}, {\"ts\": 1039, \"metric\": 58.5, \"host\": \"host-0\"}, {\"ts\": 1040, \"metric\": 60.0, \"host\": \"host-1\"}, {\"ts\": 1041, \"metric\": 61.5, \"host\": \"host-2\"}, {\"ts\": 1042, \"metric\": 63.0, \"host\": \"host-0\"}, {\"ts\": 1043, \"metric\": 64.5, \"host\": \"host-1\"}, {\"ts\": 1044, \"metric\": 66.0, \"host\": \"host-2\"}, {\"ts\": 1045, \"metric\": 67.5, \"host\": \"host-0\"}, {\"ts\": 1046, \"metric\": 69.0, \"host\": \"host-1\"}, {\"ts\": 1047, \"metric\": 70.5, \"host\": \"host-2\"}, {\"ts\": 1048, \"metric\": 72.0, \"host\": \"host-0\"}, {\"ts\": 1049, \"metric\": 73.5, \"host\": \"host-1\"}]","strategy": "skip:unique_entities_no_signal(50->50)",- "was_modified": true+ "was_modified": false},- "recorded_at": "2026-04-27T06:59:40.020402+00:00",+ "recorded_at": "2026-10-08T19:16:37.953147+00:00","transform": "smart_crusher"}diff --git a/tests/parity/fixtures/smart_crusher/unicode_dict_array_4820ed6c0c9a.json b/tests/parity/fixtures/smart_crusher/unicode_dict_array_b85509fce501.jsonsimilarity index 66%rename from tests/parity/fixtures/smart_crusher/unicode_dict_array_4820ed6c0c9a.jsonrename to tests/parity/fixtures/smart_crusher/unicode_dict_array_b85509fce501.jsonindex a4af718d2..273f0caa7 100644--- a/tests/parity/fixtures/smart_crusher/unicode_dict_array_4820ed6c0c9a.json+++ b/tests/parity/fixtures/smart_crusher/unicode_dict_array_b85509fce501.json@@ -1,11 +1,18 @@{"config": {+ "compaction_core_field_fraction": 0.8,+ "compaction_heterogeneous_core_ratio": 0.6,+ "compaction_max_buckets": 8,+ "compaction_max_flatten_inner_keys": 6,+ "compaction_min_buckets": 2,"dedup_identical_items": true,"enabled": true,"factor_out_constants": false,"first_fraction": 0.3,"include_summaries": false,"last_fraction": 0.15,+ "lossless_min_savings_ratio": 0.15,+ "lossless_only": false,"max_items_after_crush": 15,"min_items_to_analyze": 5,"min_tokens_to_crush": 200,@@ -21,14 +28,14 @@"content": "[{\"id\": 0, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 0\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 1, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 1\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 2, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 2\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 3, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 3\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 4, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 4\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 5, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 5\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 6, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 6\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 7, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 7\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 8, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 8\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 9, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 9\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 10, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 10\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 11, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 11\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 12, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 12\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 13, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 13\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 14, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 14\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 15, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 15\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 16, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 16\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 17, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 17\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 18, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 18\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 19, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 19\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}]","query": ""},- "input_sha256": "4820ed6c0c9a4f52ab17d9bd0d6e7010fea978a227b24cb2a7b0cdba5f6fe5a6",+ "input_sha256": "b85509fce50161c9b5f2e0892bfc8f6caab8e40df68cd3c6a136808d03b4cb0f","label": "unicode_dict_array","output": {- "compressed": "[{\"id\":0,\"msg\":\"hello \u4e2d\u6587 \u0440\u0443\u0441\u0441\u043a\u0438\u0439 0\",\"tag\":\"\u0442\u0435\u0441\u0442\"},{\"id\":1,\"msg\":\"hello \u4e2d\u6587 \u0440\u0443\u0441\u0441\u043a\u0438\u0439 1\",\"tag\":\"\u0442\u0435\u0441\u0442\"},{\"id\":2,\"msg\":\"hello \u4e2d\u6587 \u0440\u0443\u0441\u0441\u043a\u0438\u0439 2\",\"tag\":\"\u0442\u0435\u0441\u0442\"},{\"id\":3,\"msg\":\"hello \u4e2d\u6587 \u0440\u0443\u0441\u0441\u043a\u0438\u0439 3\",\"tag\":\"\u0442\u0435\u0441\u0442\"},{\"id\":4,\"msg\":\"hello \u4e2d\u6587 \u0440\u0443\u0441\u0441\u043a\u0438\u0439 4\",\"tag\":\"\u0442\u0435\u0441\u0442\"},{\"id\":5,\"msg\":\"hello \u4e2d\u6587 \u0440\u0443\u0441\u0441\u043a\u0438\u0439 5\",\"tag\":\"\u0442\u0435\u0441\u0442\"},{\"id\":6,\"msg\":\"hello \u4e2d\u6587 \u0440\u0443\u0441\u0441\u043a\u0438\u0439 6\",\"tag\":\"\u0442\u0435\u0441\u0442\"},{\"id\":7,\"msg\":\"hello \u4e2d\u6587 \u0440\u0443\u0441\u0441\u043a\u0438\u0439 7\",\"tag\":\"\u0442\u0435\u0441\u0442\"},{\"id\":8,\"msg\":\"hello \u4e2d\u6587 \u0440\u0443\u0441\u0441\u043a\u0438\u0439 8\",\"tag\":\"\u0442\u0435\u0441\u0442\"},{\"id\":9,\"msg\":\"hello \u4e2d\u6587 \u0440\u0443\u0441\u0441\u043a\u0438\u0439 9\",\"tag\":\"\u0442\u0435\u0441\u0442\"},{\"id\":10,\"msg\":\"hello \u4e2d\u6587 \u0440\u0443\u0441\u0441\u043a\u0438\u0439 10\",\"tag\":\"\u0442\u0435\u0441\u0442\"},{\"id\":11,\"msg\":\"hello \u4e2d\u6587 \u0440\u0443\u0441\u0441\u043a\u0438\u0439 11\",\"tag\":\"\u0442\u0435\u0441\u0442\"},{\"id\":12,\"msg\":\"hello \u4e2d\u6587 \u0440\u0443\u0441\u0441\u043a\u0438\u0439 12\",\"tag\":\"\u0442\u0435\u0441\u0442\"},{\"id\":13,\"msg\":\"hello \u4e2d\u6587 \u0440\u0443\u0441\u0441\u043a\u0438\u0439 13\",\"tag\":\"\u0442\u0435\u0441\u0442\"},{\"id\":14,\"msg\":\"hello \u4e2d\u6587 \u0440\u0443\u0441\u0441\u043a\u0438\u0439 14\",\"tag\":\"\u0442\u0435\u0441\u0442\"},{\"id\":15,\"msg\":\"hello \u4e2d\u6587 \u0440\u0443\u0441\u0441\u043a\u0438\u0439 15\",\"tag\":\"\u0442\u0435\u0441\u0442\"},{\"id\":16,\"msg\":\"hello \u4e2d\u6587 \u0440\u0443\u0441\u0441\u043a\u0438\u0439 16\",\"tag\":\"\u0442\u0435\u0441\u0442\"},{\"id\":17,\"msg\":\"hello \u4e2d\u6587 \u0440\u0443\u0441\u0441\u043a\u0438\u0439 17\",\"tag\":\"\u0442\u0435\u0441\u0442\"},{\"id\":18,\"msg\":\"hello \u4e2d\u6587 \u0440\u0443\u0441\u0441\u043a\u0438\u0439 18\",\"tag\":\"\u0442\u0435\u0441\u0442\"},{\"id\":19,\"msg\":\"hello \u4e2d\u6587 \u0440\u0443\u0441\u0441\u043a\u0438\u0439 19\",\"tag\":\"\u0442\u0435\u0441\u0442\"}]",+ "compressed": "[{\"id\": 0, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 0\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 1, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 1\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 2, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 2\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 3, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 3\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 4, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 4\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 5, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 5\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 6, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 6\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 7, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 7\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 8, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 8\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 9, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 9\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 10, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 10\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 11, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 11\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 12, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 12\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 13, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 13\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 14, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 14\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 15, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 15\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 16, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 16\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 17, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 17\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 18, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 18\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 19, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 19\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}]","original": "[{\"id\": 0, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 0\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 1, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 1\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 2, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 2\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 3, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 3\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 4, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 4\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 5, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 5\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 6, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 6\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 7, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 7\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 8, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 8\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 9, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 9\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 10, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 10\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 11, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 11\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 12, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 12\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 13, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 13\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 14, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 14\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 15, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 15\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 16, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 16\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 17, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 17\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 18, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 18\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 19, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 19\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}]","strategy": "skip:unique_entities_no_signal(20->20)",- "was_modified": true+ "was_modified": false},- "recorded_at": "2026-04-27T06:59:39.970946+00:00",+ "recorded_at": "2026-10-08T19:16:37.874112+00:00","transform": "smart_crusher"}diff --git a/tests/parity/record_smart_crusher.py b/tests/parity/record_smart_crusher.pyindex e8e24a7fe..3e5549479 100644--- a/tests/parity/record_smart_crusher.py+++ b/tests/parity/record_smart_crusher.py@@ -33,11 +33,13 @@ from __future__ import annotationsimport datetime as _dtimport hashlib+import inspectimport jsonfrom dataclasses import asdictfrom pathlib import Pathfrom typing import Any+from headroom._core import SmartCrusherConfig as NativeSmartCrusherConfigfrom headroom.transforms.smart_crusher import SmartCrusher, SmartCrusherConfig_REPO_ROOT = Path(__file__).resolve().parent.parent.parent@@ -57,11 +59,14 @@ def _record(config: SmartCrusherConfig | None = None,) -> Path:cfg = config or SmartCrusherConfig()- crusher = SmartCrusher(config=cfg)+ # Match both parity comparators' legacy keep/drop mode.+ crusher = SmartCrusher(config=cfg, with_compaction=False)result = crusher.crush(content, query=query, bias=bias)payload_input = {"content": content, "query": query, "bias": bias}- payload_config = asdict(cfg)+ # Fixtures are replayed through PyO3, not the wrapper-only guard options.+ native_parameters = inspect.signature(NativeSmartCrusherConfig).parameters+ payload_config = {key: value for key, value in asdict(cfg).items() if key in native_parameters}payload_output = {"compressed": result.compressed,"original": result.original,@@ -89,7 +94,7 @@ def _record(_FIXTURES_DIR.mkdir(parents=True, exist_ok=True)target = _FIXTURES_DIR / f"{label}_{digest[:12]}.json"- target.write_text(json.dumps(fixture, indent=2, sort_keys=True) + "\n")+ target.write_text(json.dumps(fixture, indent=2, sort_keys=True) + "\n", newline="\n")return target@@ -103,25 +108,11 @@ def _scenarios() -> list[tuple[str, str, str, float]]:# unchanged; trivially byte-equal.out.append(("non_json_passthrough", "this is not json at all", "", 1.0))- # 2. JSON object with no array fields long enough to crush.- out.append(- (- "small_object_passthrough",- json.dumps({"a": 1, "b": 2, "c": "hello"}),- "",- 1.0,- )- )+ # 2. An object without crushable arrays retains its whitespace.+ out.append(("small_object_passthrough", json.dumps({"a": 1, "b": 2, "c": "hello"}), "", 1.0))- # 3. Short array (below min_items_to_analyze=5) → passthrough.- out.append(- (- "short_array_passthrough",- json.dumps([1, 2, 3]),- "",- 1.0,- )- )+ # 3. An array below the analysis threshold is byte-identical.+ out.append(("short_array_passthrough", json.dumps([1, 2, 3]), "", 1.0))# 4. Dict array with 30 items, varied integer status field.# Exercises crush_array's adaptive_k → smart_sample / top_n path.@@ -142,7 +133,8 @@ def _scenarios() -> list[tuple[str, str, str, float]]:mixed_arr = ["start"] + list(range(20)) + ["middle"] + ["end"] * 5out.append(("mixed_array", json.dumps(mixed_arr), "", 1.0))- # 8. Nested: top-level dict whose `events` field is a long dict array.+ # Nested arrays whose analysis keeps every item must preserve their+ # enclosing object's original bytes rather than reserialize it.nested = {"request_id": "req-1","events": [{"step": i, "kind": "trace", "msg": f"e{i}"} for i in range(20)],@@ -155,9 +147,7 @@ def _scenarios() -> list[tuple[str, str, str, float]]:# 10. Bias < 1 (keep fewer) on the 30-dict case.out.append(("dict_array_30_bias_low", json.dumps(items_30_dict), "", 0.7))- # 11. Unicode payload — exercises the `ensure_ascii=False` path in- # Python's safe_json_dumps. Rust's python_safe_json_dumps must emit- # raw UTF-8 bytes here, not `\uXXXX` escapes.+ # Preserve escaped Unicode in unchanged input, including whitespace.unicode_items = [{"id": i, "msg": f"hello 中文 русский {i}", "tag": "тест"} for i in range(20)]out.append(("unicode_dict_array", json.dumps(unicode_items), "", 1.0))@@ -169,7 +159,8 @@ def _scenarios() -> list[tuple[str, str, str, float]]:]out.append(("dict_array_100_sequential", json.dumps(big_seq), "", 1.0))- # 13. Time-series-like payload: monotonic timestamp + float metric.+ # Time-series analysis may keep every point; its float spellings and+ # original JSON formatting must then survive the Python/native bridge.ts = [{"ts": 1000 + i, "metric": float(i * 1.5), "host": f"host-{i % 3}"} for i in range(50)]out.append(("time_series_50", json.dumps(ts), "", 1.0))@@ -180,18 +171,12 @@ def _scenarios() -> list[tuple[str, str, str, float]]:# 15. Empty array — boundary case, must round-trip cleanly.out.append(("empty_array", json.dumps([]), "", 1.0))- # 16. Array of nulls and bools — non-crushable mixed type.+ # 16. Non-crushable null/bool mixtures retain their original separators.out.append(- (- "nulls_and_bools",- json.dumps([None, True, False, None, True, False, None]),- "",- 1.0,- )+ ("nulls_and_bools", json.dumps([None, True, False, None, True, False, None]), "", 1.0))- # 17. Deeply nested structure: 3-level depth with arrays at each- # level. Exercises process_value's recursion.+ # Recursive no-op analysis must preserve bytes through every object level.deep = {"a": {"b": {"events": [{"i": i, "kind": "deep", "v": f"x{i}"} for i in range(15)]}}}out.append(("nested_3deep_with_array", json.dumps(deep), "", 1.0))diff --git a/tests/test_code_aware_regressions.py b/tests/test_code_aware_regressions.pyindex 2dbcc2c11..65e8ee901 100644--- a/tests/test_code_aware_regressions.py+++ b/tests/test_code_aware_regressions.py@@ -114,7 +114,7 @@ def test_content_router_code_aware_token_accounting_matches_router_estimate() ->router = ContentRouter(ContentRouterConfig(enable_code_aware=True))router._code_compressor = _FakeCodeCompressor(compressed_text)- compressed, compressed_tokens, strategy_chain = router._apply_strategy_to_content(+ compressed, compressed_tokens, strategy_chain, _reversible = router._apply_strategy_to_content(original, CompressionStrategy.CODE_AWARE, context="")diff --git a/tests/test_integrations/langchain/test_langchain_1x_compat.py b/tests/test_integrations/langchain/test_langchain_1x_compat.pyindex a294a0333..43d81f8c1 100644--- a/tests/test_integrations/langchain/test_langchain_1x_compat.py+++ b/tests/test_integrations/langchain/test_langchain_1x_compat.py@@ -33,16 +33,28 @@ except ImportError:pytestmark = pytest.mark.skipif(not LANGCHAIN_AVAILABLE, reason="LangChain not installed")+# Repeated rows require a real transformation, not serialization-only savings.BIG_RESULT = json.dumps({"results": [- {"id": i, "user": f"user{i}", "plan": "pro", "status": "active"} for i in range(300)+ {"id": i % 10, "user": f"user{i % 10}", "plan": "pro", "status": "active"}+ for i in range(300)],"total": 300,})+def _assert_compressed_rows(output: str) -> None:+ original = json.loads(BIG_RESULT)+ compressed = json.loads(output)+ kept = [row for row in compressed["results"] if "id" in row]+ assert compressed["total"] == original["total"]+ assert {row["id"]: row for row in kept} == {row["id"]: row for row in original["results"]}+ assert len(kept) < len(original["results"])+ assert len(output) < len(BIG_RESULT)++@tooldef query_database(query: str) -> str:"""Query the users database. Returns JSON rows."""@@ -110,8 +122,7 @@ class TestWrappedToolIsUsable:wrapped = wrap_tools_with_headroom([query_database], min_chars_to_compress=1000)[0]out = wrapped.invoke({"query": "signups"})- assert isinstance(out, str) and out- assert len(out) < len(BIG_RESULT)+ _assert_compressed_rows(out)def test_argument_schema_is_preserved(self):from headroom.integrations import wrap_tools_with_headroom@@ -129,7 +140,7 @@ class TestWrappedToolIsUsable:wrapped = wrap_tools_with_headroom([query_database], min_chars_to_compress=1000)[0]out = asyncio.run(wrapped.ainvoke({"query": "signups"}))- assert len(out) < len(BIG_RESULT)+ _assert_compressed_rows(out)def test_unusable_args_schema_falls_back_to_inference(self):"""A tool carrying a schema LangChain cannot use must still wrap."""diff --git a/tests/test_lossless_mode.py b/tests/test_lossless_mode.pyindex ba6c80e45..7de1265ac 100644--- a/tests/test_lossless_mode.py+++ b/tests/test_lossless_mode.py@@ -252,7 +252,7 @@ def test_router_lossless_log_strategy_no_marker_and_recoverable() -> None:+ "connection refused by upstream service alpha\n" * 40+ "[info] boot sequence complete\n")- out, _tokens, chain = router._apply_strategy_to_content(+ out, _tokens, chain, _reversible = router._apply_strategy_to_content(log, CompressionStrategy.LOG, context="")_assert_no_marker(out)@@ -273,7 +273,7 @@ def test_router_lossless_diff_strategy_no_marker() -> None:"-old\n""+new\n")- out, _tokens, chain = router._apply_strategy_to_content(+ out, _tokens, chain, _reversible = router._apply_strategy_to_content(diff, CompressionStrategy.DIFF, context="")_assert_no_marker(out)diff --git a/tests/test_openai_responses_context_compaction.py b/tests/test_openai_responses_context_compaction.pyindex 16b2f8741..059a1801f 100644--- a/tests/test_openai_responses_context_compaction.py+++ b/tests/test_openai_responses_context_compaction.py@@ -355,7 +355,7 @@ def test_content_router_retries_kompress_when_structured_strategy_noops(monkeypalambda value, context, question=None: ("short summary", 2),)- compressed, compressed_tokens, strategy_chain = router._apply_strategy_to_content(+ compressed, compressed_tokens, strategy_chain, _reversible = router._apply_strategy_to_content(content,CompressionStrategy.SMART_CRUSHER,context="",diff --git a/tests/test_proxy_ccr.py b/tests/test_proxy_ccr.pyindex 4cb122815..ba3307081 100644--- a/tests/test_proxy_ccr.py+++ b/tests/test_proxy_ccr.py@@ -363,15 +363,14 @@ class TestEndToEndTOINIntegration:from headroom.telemetry import ToolSignaturefrom headroom.transforms import SmartCrusher, TransformPipeline- # Create tool output with 100 items that will trigger compression- # Key: score field with varying values signals sortable data- # Having repetitive category values helps trigger compression+ # Duplicate ranked rows require real keep/drop compression, so+ # recording TOIN and caching originals cannot pass on JSON formatting.items = [{- "id": i,- "score": 1000 - i, # Decreasing scores signal sorting- "category": f"cat_{i % 3}", # Only 3 unique categories- "status": "active" if i % 2 == 0 else "inactive", # Binary status+ "id": i % 5,+ "score": 1000 - (i % 5),+ "category": f"cat_{i % 5}",+ "status": "active",}for i in range(100)]@@ -434,40 +433,26 @@ class TestEndToEndTOINIntegration:model_limit=200000,)- # Verify SmartCrusher was invoked (transform name starts with smart_crush)- smart_crush_applied = any(- t.startswith("smart_crush") or t.startswith("smart:") for t in result.transforms_applied- )- assert smart_crush_applied, (- f"SmartCrusher should be in transforms: {result.transforms_applied}"- )+ output = result.messages[2]["content"][0]["content"]+ # The pipeline appends its retrieval notice after the JSON payload.+ compressed, _ = json.JSONDecoder().raw_decode(output)+ kept = [row for row in compressed if "id" in row]+ assert len(kept) < len(items)+ assert all(row in items for row in kept)- # Check if compression was actually performed (not skipped)- # Skip messages look like "smart:skip:reason(100->100)"- compression_was_skipped = any(- "skip" in t.lower() for t in result.transforms_applied if "smart:" in t.lower()- )+ store = get_compression_store()+ marker_text = compressed[-1]["_ccr_dropped"]+ ccr_hash = marker_text[len("<<ccr:") :].split(" ", 1)[0]+ entry = store.retrieve(ccr_hash)+ assert entry is not None, "The rendered CCR marker must be retrievable"+ assert json.loads(entry.original_content) == items- # If compression happened, verify TOIN and store- if not compression_was_skipped:- # Verify compression store has the entry- store = get_compression_store()- stats = store.get_stats()- assert stats["entry_count"] >= 1, "Should have cached entry"-- # Verify TOIN recorded the compression- signature = ToolSignature.from_items(items)- pattern = fresh_toin._patterns.get(signature.structure_hash)- assert pattern is not None, (- "TOIN should have recorded compression event. "- "If this fails, SmartCrusher is not calling TOIN.record_compression."- )- assert pattern.total_compressions >= 1, "Should have at least 1 compression"- else:- # Compression was skipped - this is expected for some data patterns- # The important thing is that SmartCrusher was invoked and made a decision- # The other tests verify the full loop when compression does happen- pass+ signature = ToolSignature.from_items(items)+ pattern = fresh_toin.get_pattern(signature.structure_hash)+ assert pattern is not None, "TOIN should have recorded the real compression"+ assert pattern.total_compressions == 1+ assert pattern.total_items_seen == len(items)+ assert pattern.total_items_kept == len(kept)def test_retrieval_through_proxy_updates_toin_field_semantics(self, fresh_toin, client_with_optimizationdiff --git a/tests/test_proxy_compress_endpoint.py b/tests/test_proxy_compress_endpoint.pyindex b6456497b..1a99fb75c 100644--- a/tests/test_proxy_compress_endpoint.py+++ b/tests/test_proxy_compress_endpoint.py@@ -437,7 +437,7 @@ class TestCompressEndpointLossyInlineMode:NO CCR marker / retrieval round-trip, so the output is safe to forwardstraight to a provider (Kong-sidecar use case)."""- def _big_tool_message(self):+ def _big_tool_message(self, repetitions=1):large_data = json.dumps([{@@ -448,8 +448,9 @@ class TestCompressEndpointLossyInlineMode:f"category=electronics, price={i * 10.99:.2f}, stock={i * 5}.","tags": ["electronics", "sale", "featured", "new-arrival"],}- for i in range(200)+ for i in range(200 // repetitions)]+ * repetitions)return [{"role": "user", "content": "What items are available?"},@@ -459,9 +460,7 @@ class TestCompressEndpointLossyInlineMode:@pytest.fixturedef client(self):- # disable_kompress keeps the real ONNX model out of the test: marker- # suppression is exercised by SmartCrusher (pure-Python) regardless, and- # the mode inherits enable_kompress from config so this stays fast.+ # Exercise the actual native JSON steps without unrelated ML inference.config = ProxyConfig(optimize=True,cache_enabled=False,@@ -473,8 +472,11 @@ class TestCompressEndpointLossyInlineMode:with TestClient(app, base_url="http://127.0.0.1", client=("127.0.0.1", 12345)) as c:yield c- def test_lossy_inline_emits_no_ccr_markers(self, client):- messages = self._big_tool_message()+ @pytest.mark.parametrize(+ ("repetitions", "strategy"), [(1, "json_minifier"), (20, "smart_crusher")]+ )+ def test_lossy_inline_emits_no_ccr_markers(self, client, repetitions, strategy):+ messages = self._big_tool_message(repetitions=repetitions)response = client.post("/v1/compress",json={"messages": messages, "model": "gpt-4", "config": {"mode": "lossy_inline"}},@@ -497,6 +499,37 @@ class TestCompressEndpointLossyInlineMode:assert data["tokens_after"] < data["tokens_before"]assert data["tokens_saved"] == data["tokens_before"] - data["tokens_after"]+ original_rows = json.loads(messages[1]["content"])+ tool = next(message for message in data["messages"] if message["role"] == "tool")+ compressed_rows, _ = json.JSONDecoder().raw_decode(tool["content"])+ if repetitions == 1:+ assert compressed_rows == original_rows+ else:+ assert len(compressed_rows) < len(original_rows)+ assert {json.dumps(row, sort_keys=True) for row in compressed_rows} == {+ json.dumps(row, sort_keys=True) for row in original_rows+ }+ assert strategy in {transform.split(":")[1] for transform in data["transforms_applied"]}++ def test_lossy_inline_preserves_no_op_tool_content(self, client):+ messages = self._big_tool_message()+ messages[1]["content"] = json.dumps(+ json.loads(messages[1]["content"]), separators=(",", ":")+ )+ original = messages[1]["content"]+ response = client.post(+ "/v1/compress",+ json={"messages": messages, "model": "gpt-4", "config": {"mode": "lossy_inline"}},+ )+ assert response.status_code == 200+ data = response.json()++ tool = next(message for message in data["messages"] if message["role"] == "tool")+ assert tool["content"] == original+ assert data["tokens_saved"] == 0+ assert data["tokens_after"] == data["tokens_before"]+ assert data["ccr_hashes"] == []+def test_lossless_then_lossy_alias(self, client):"""The spelled-out alias selects the same mode."""messages = self._big_tool_message()diff --git a/tests/test_recursive_json.py b/tests/test_recursive_json.pyindex 53a8ff8cb..01275ada3 100644--- a/tests/test_recursive_json.py+++ b/tests/test_recursive_json.py@@ -1,6 +1,4 @@-"""Unit tests for headroom.transforms.recursive_json — the structural (embedded)-JSON routing step. Uses a fake dispatch so the mechanism is tested in isolation-from the real compressors."""+"""Embedded JSON routing: preserved surroundings and per-adopted-span recovery."""from __future__ import annotations@@ -13,62 +11,89 @@ from headroom.transforms.recursive_json import ()-def _upper_dispatch(span: str) -> str | None:- """Fake compressor: returns a shorter deterministic stand-in for any span."""- try:- v = json.loads(span)- except ValueError:- return None- return f"<TABLE n={len(v)}>" if isinstance(v, list) else "<OBJ>"+def _minify_dispatch(span: str) -> tuple[str, bool]:+ return json.dumps(json.loads(span), separators=(",", ":")), Truedef test_embedded_json_routed_and_surroundings_exact() -> None:- payload = json.dumps([{"id": i, "ok": True} for i in range(6)], separators=(",", ":"))+ rows = [{"id": i, "ok": True} for i in range(6)]+ payload = json.dumps(rows, indent=2)content = f"Fetched rows from API:\n{payload}\nDone (200 OK)."- out = route_embedded_json(content, _upper_dispatch)- assert out is not None- assert out.startswith("Fetched rows from API:\n")- assert out.endswith("\nDone (200 OK).")- assert "<TABLE n=6>" in out+ assert route_embedded_json(content, _minify_dispatch) == (+ f"Fetched rows from API:\n{json.dumps(rows, separators=(',', ':'))}\nDone (200 OK).",+ True,+ )+++def test_noop_json_span_does_not_block_later_compaction() -> None:+ compact = json.dumps([{"id": i, "ok": True} for i in range(6)], separators=(",", ":"))+ rows = [{"id": i, "message": f"entry-{i}"} for i in range(20, 26)]+ padded = json.dumps(rows, indent=2)+ leading, between, trailing = "Début 🚀\r\nfirst: ", "\r\nsecond: ", "\r\nFin."+ content = leading + compact + between + padded + trailing++ assert route_embedded_json(content, _minify_dispatch) == (+ leading + compact + between + json.dumps(rows, separators=(",", ":")) + trailing,+ True,+ )def test_ccr_marker_span_passed_through() -> None:- # A span already carrying a CCR marker must never be re-routed (R1).content = 'prefix [{"a":1,"b":2},{"a":3,"b":"<<ccr:deadbeef,json,900>>"}] suffix'- out = route_embedded_json(content, _upper_dispatch)- assert out is None # only span contains a marker → skipped → nothing to do+ assert route_embedded_json(content, _minify_dispatch) is Nonedef test_no_json_is_noop() -> None:- assert route_embedded_json("just prose, nothing structured here", _upper_dispatch) is None+ assert route_embedded_json("just prose, nothing structured here", _minify_dispatch) is Nonedef test_whole_block_json_is_callers_job() -> None:- # A block that IS a single JSON value is skipped (routed by the caller).content = json.dumps([{"a": i} for i in range(5)], separators=(",", ":"))- assert route_embedded_json(content, _upper_dispatch) is None+ assert route_embedded_json(content, _minify_dispatch) is Nonedef test_benefit_gate_declines_when_not_smaller() -> None:- payload = json.dumps([{"a": i} for i in range(5)], separators=(",", ":"))- content = f"x {payload} y"- # Dispatch that returns something LARGER → must be declined (outcome gate).- assert route_embedded_json(content, lambda s: s + " " * 999) is None+ payload = json.dumps([{"a": i} for i in range(5)])+ assert route_embedded_json(f"x {payload} y", lambda s: (s + " " * 999, True)) is None+++def test_unmarked_loss_cannot_borrow_another_spans_marker() -> None:+ first = json.dumps([{"id": i} for i in range(10)], indent=2)+ second = json.dumps([{"value": i} for i in range(10)], indent=2)++ def dispatch(span: str) -> tuple[str, bool]:+ rows = json.loads(span)+ if "id" in rows[0]:+ return "<<ccr:deadbeef,json,100>>", True+ return json.dumps(rows[:1]), False++ for left, right in ((first, second), (second, first)):+ result = route_embedded_json(f"first: {left}\nsecond: {right}", dispatch, tok=len)+ assert result is not None+ text, reversible = result+ assert "<<ccr:deadbeef,json,100>>" in text+ assert '{"value": 0}' in text and '{"value": 9}' not in text+ assert reversible is False+++def test_rejected_lossy_candidate_does_not_taint_adopted_minification() -> None:+ first = json.dumps([{"id": i} for i in range(10)], indent=2)+ second = json.dumps([{"value": i} for i in range(10)], indent=2)+ def dispatch(span: str) -> tuple[str, bool]:+ rows = json.loads(span)+ if "id" in rows[0]:+ return json.dumps(rows[:1]) + " " * len(span), False+ return _minify_dispatch(span)-def test_deterministic() -> None:- payload = json.dumps([{"k": i} for i in range(8)], separators=(",", ":"))- content = f"a {payload} b {payload} c"- r1 = route_embedded_json(content, _upper_dispatch)- r2 = route_embedded_json(content, _upper_dispatch)- assert r1 == r2 and r1 is not None- assert r1.count("<TABLE n=8>") == 2 # both embedded spans routed+ assert route_embedded_json(f"first: {first}\nsecond: {second}", dispatch, tok=len) == (+ f"first: {first}\nsecond: {json.dumps(json.loads(second), separators=(',', ':'))}",+ True,+ )def test_scalar_array_not_routed() -> None:- # array of scalars is not a "routable" JSON shape (no dict rows)- content = "nums: [1,2,3,4,5,6,7,8] done"- assert route_embedded_json(content, _upper_dispatch) is None+ assert route_embedded_json("nums: [1,2,3,4,5,6,7,8] done", _minify_dispatch) is Nonedef test_json_document_spans_finds_containers_anywhere() -> None:diff --git a/tests/test_relevance_split.py b/tests/test_relevance_split.pyindex 6f7e5ef5b..3e2e57132 100644--- a/tests/test_relevance_split.py+++ b/tests/test_relevance_split.py@@ -215,7 +215,9 @@ def test_router_lossless_mode_folds_only_no_drop():# irrelevant "widget" records must be PRESERVED, not silently dropped, and the# Kompress tail stub must never run.r = _router(split_on=True) # lossless mode- out, _, chain = r._apply_strategy_to_content(_SEARCH, CompressionStrategy.SEARCH, "oauth token")+ out, _, chain, _reversible = r._apply_strategy_to_content(+ _SEARCH, CompressionStrategy.SEARCH, "oauth token"+ )assert chain == ["lossless_search"]assert "oauth token" in out # relevant records keptassert "widget" in out # irrelevant tail ALSO kept — no silent drop in lossless mode@@ -228,7 +230,9 @@ def test_router_relevance_split_fires_in_ccr_mode():# #1721); the _try_ml_compressor stub stands in for it here. Proves the# split is mode-agnostic, not lossless-only.r = _router(split_on=True, lossless=False)- out, _, chain = r._apply_strategy_to_content(_SEARCH, CompressionStrategy.SEARCH, "oauth token")+ out, _, chain, _reversible = r._apply_strategy_to_content(+ _SEARCH, CompressionStrategy.SEARCH, "oauth token"+ )assert chain == ["search", "relevance_split"]assert "oauth token" in outassert "[TAIL]" in out@@ -237,14 +241,18 @@ def test_router_relevance_split_fires_in_ccr_mode():def test_router_diff_stays_pure_lossless():r = _router(split_on=True)diff = "diff --git a/x b/x\nindex 111..222 100644\n@@ -1 +1 @@\n-old widget\n+new oauth token\n"- _, _, chain = r._apply_strategy_to_content(diff, CompressionStrategy.DIFF, "oauth token")+ _, _, chain, _reversible = r._apply_strategy_to_content(+ diff, CompressionStrategy.DIFF, "oauth token"+ )assert "relevance_split" not in chain # Kompressing hunks would break applyassert chain == ["lossless_diff"]def test_router_split_can_be_disabled():r = _router(split_on=False)- _, _, chain = r._apply_strategy_to_content(_SEARCH, CompressionStrategy.SEARCH, "oauth token")+ _, _, chain, _reversible = r._apply_strategy_to_content(+ _SEARCH, CompressionStrategy.SEARCH, "oauth token"+ )assert "relevance_split" not in chaindiff --git a/tests/test_router_external_dispatch.py b/tests/test_router_external_dispatch.pyindex 47b2a66da..27ee1ed15 100644--- a/tests/test_router_external_dispatch.py+++ b/tests/test_router_external_dispatch.py@@ -33,7 +33,6 @@ from headroom.transforms.compressor_registry import (CompressOutput,)from headroom.transforms.content_router import (- CompressionStrategy,ContentRouter,ContentRouterConfig,)@@ -250,21 +249,6 @@ def test_native_misclassification_keeps_git_status_format(_memory_ccr, monkeypatassert detected.metadata["format"] == "git_status"-def test_external_dispatch_via_apply_strategy_returns_normal_shape(_memory_ccr):- comp = _RecordingExternal(name="ext_json", content_types=("application/json",))- router = _router_with_external(comp, ["ext_json"])-- compressed, tokens, chain = router._apply_strategy_to_content(- _JSON_ARRAY, CompressionStrategy.SMART_CRUSHER, ""- )-- assert chain == ["external:ext_json"]- assert "external-compressed" in compressed- # Tokens counted with the router's OWN estimator (a positive int), not the- # compressor's self-report.- assert isinstance(tokens, int) and tokens > 0--# ─────────────────────────── (b) recoverable map ─────────────────────────────diff --git a/tests/test_router_registry_dispatch.py b/tests/test_router_registry_dispatch.pyindex 92257a105..097409a64 100644--- a/tests/test_router_registry_dispatch.py+++ b/tests/test_router_registry_dispatch.py@@ -1,360 +1,25 @@-"""Byte-identical differential tests for registry-resolved built-in dispatch.+"""Unavailable HTML extraction must preserve tool output."""-The content router now dispatches the SIMPLE built-in strategies (CONFIG, LOG,-SEARCH, TABULAR) through the compressor registry instead of a hardcoded direct-``self._get_*().compress(...)`` call in ``_apply_strategy_to_content``. Each-built-in adapter delegates to the SAME ``_get_*`` getter+method with the SAME-arguments, so registry-resolved dispatch must be byte-identical to the historical-direct dispatch: same compressed content, same token count, same single-entry-``strategy_chain``.--Each FLIPPED strategy has a differential test comparing the router's dispatch-output to the built-in's direct output obtained via its ``_get_*`` getter — i.e.-"registry dispatch == old dispatch". SMART_CRUSHER is now also flipped (its-``.crush`` primary invocation goes through the registry while the shared-Kompress→Log fallback block stays direct); KOMPRESS/TEXT now dispatch via the-registry too — the ``kompress`` adapter delegates to the SAME ``_try_ml_compressor``-call with ``question`` forwarded via ``config['question']``, so CONTENT is-preserved and only the reported token metric changed. See-``test_router_registry_smartcrusher.py`` for the full SMART_CRUSHER fallback-chain-and KOMPRESS/TEXT differential coverage.--Offline guardrails:- * No real ML/ONNX/HF inference — the KOMPRESS ML boundary is mocked.- * The flipped strategies shrink their representative content, so no zero-savings- Kompress fallback fires (that would touch the ML boundary and append KOMPRESS- to the chain).- * ``lossless_then_lossy`` and ``relevance_split`` are off so the if/elif branch- is the terminal path; STAGE 0 (``_lossless_first``) is neutralized so search/- log folds don't return before the branch. Both are shared, unchanged code.- * The broad ``content_router``/``compression`` -k selection is NOT exercised- (it hangs on HF-Hub/ONNX).-"""--from __future__ import annotations--import jsonfrom types import SimpleNamespaceimport pytestfrom headroom.transforms.content_router import (- _BUILTIN_COMPRESSOR_DESCRIPTORS,CompressionStrategy,ContentRouter,ContentRouterConfig,- _estimate_tokens,)-def _router() -> ContentRouter:- """Router whose if/elif branch is the terminal dispatch path.-- ``relevance_split`` off (no LOG/SEARCH relevance split) and- ``lossless_then_lossy`` off (no lossy layer on top of a strategy result) so a- successful strategy result returns directly. ``ccr_inject_marker`` off makes- the compressed output deterministic and marker-free; it is applied identically- to the direct reference and the dispatch router, so the differential holds- regardless of its value.- """- return ContentRouter(- ContentRouterConfig(- relevance_split=False,- lossless_then_lossy=False,- ccr_inject_marker=False,- )- )---def _isolate_branch(monkeypatch: pytest.MonkeyPatch, router: ContentRouter) -> None:- """Neutralize STAGE 0 so the if/elif branch under test is exercised.-- ``_lossless_first`` runs unconditionally and can fold search/log content,- returning before the if/elif. It is shared, unchanged code (the flip only- touches the branch bodies), so forcing it to a no-op isolates what the flip- actually changed without altering the branch semantics.- """- monkeypatch.setattr(router, "_lossless_first", lambda content, strategy: (content, None))---# Representative content per type. The flipped strategies must SHRINK this so the-# fallback-eligible strategies (TABULAR/CONFIG) don't trip the zero-savings-# Kompress fallback (which would append KOMPRESS to the chain).-_SEARCH = "\n".join(f"src/file{i}.py:{i}: def func{i}(): return {i}" for i in range(30))-_LOG = (- "\n".join(f"2024-01-01 12:00:{i:02d} INFO task {i}" for i in range(30))- + "\n"- + "\n".join("identical repeated line" for _ in range(25))-)-# A markdown table the tabular compressor actually shrinks (schema-fold), and a-# repetitive INI the config compressor actually shrinks (block-fold) — so these-# fallback-eligible strategies produce a real token saving and the shared-# zero-savings Kompress fallback does NOT fire (chain stays single-entry).-_TABULAR = "| id | name | status | score |\n|----|------|--------|-------|\n" + "\n".join(- f"| {i} | row{i} | ok | {i * 3} |" for i in range(60)-)-_CONFIG = "\n".join(- f"[section_{i}]\nname = svc{i}\ntimeout = 30\nretries = 3\nverbose = false\nregion = us-east-1"- for i in range(40)-)---# ───────────────────────── flipped (registry dispatch) ────────────────────────---def test_search_router_dispatch_matches_direct(monkeypatch: pytest.MonkeyPatch) -> None:- router = _router()- _isolate_branch(monkeypatch, router)- context, bias = "func", 1.0- # OLD dispatch reference: same getter + method the branch used before the flip.- direct = (- _router()._get_search_compressor().compress(_SEARCH, context=context, bias=bias).compressed- )- out, tokens, chain = router._apply_strategy_to_content(- _SEARCH, CompressionStrategy.SEARCH, context, bias=bias- )- assert out == direct- assert tokens == _estimate_tokens(direct)- assert chain == [CompressionStrategy.SEARCH.value]---def test_log_router_dispatch_matches_direct(monkeypatch: pytest.MonkeyPatch) -> None:- router = _router()- _isolate_branch(monkeypatch, router)- bias = 1.0- direct = _router()._get_log_compressor().compress(_LOG, bias=bias).compressed- out, tokens, chain = router._apply_strategy_to_content(- _LOG, CompressionStrategy.LOG, "", bias=bias- )- assert out == direct- assert tokens == _estimate_tokens(direct)- assert chain == [CompressionStrategy.LOG.value]---def test_tabular_router_dispatch_matches_direct(monkeypatch: pytest.MonkeyPatch) -> None:- router = _router()- _isolate_branch(monkeypatch, router)- context, bias = "q", 1.0- direct = (- _router()- ._get_tabular_compressor()- .compress(_TABULAR, context=context, bias=bias)- .compressed- )- out, tokens, chain = router._apply_strategy_to_content(- _TABULAR, CompressionStrategy.TABULAR, context, bias=bias- )- assert out == direct- assert tokens == _estimate_tokens(direct)- # Fallback-eligible, but a real shrink means no zero-savings Kompress fallback.- assert chain == [CompressionStrategy.TABULAR.value]- assert len(out) < len(_TABULAR)---def test_config_router_dispatch_matches_direct(monkeypatch: pytest.MonkeyPatch) -> None:- router = _router()- _isolate_branch(monkeypatch, router)- context, bias = "q", 1.0- direct = (- _router()._get_config_compressor().compress(_CONFIG, context=context, bias=bias).compressed- )- out, tokens, chain = router._apply_strategy_to_content(- _CONFIG, CompressionStrategy.CONFIG, context, bias=bias- )- assert out == direct- # CONFIG's historical metric was len(text.split()) — a WORD count, which this- # assertion was written to preserve across the registry-dispatch flip. That- # was right for the flip and wrong as a metric: the value is divided by- # `original_tokens`, which comes from _estimate_tokens, so a word numerator- # over a token denominator made a byte-identical no-op score ~0.36 and get- # accepted as a large saving. Now measured in the denominator's unit.- assert tokens == _estimate_tokens(direct)- assert chain == [CompressionStrategy.CONFIG.value]- assert len(out) < len(_CONFIG)---# ─────────────────────────── deferred (unchanged) ────────────────────────────---def test_smart_crusher_router_dispatch_matches_direct(monkeypatch: pytest.MonkeyPatch) -> None:- # SMART_CRUSHER is now FLIPPED: its primary ``.crush`` invocation routes- # through the registry "smart_crusher" adapter (which delegates to the SAME- # getter + ``.crush(query=..., bias=...)``), while the shared Kompress→Log- # fallback block stays direct. Registry dispatch must be byte-identical to the- # historical direct crush. A JSON array shrinks, so no fallback fires and the- # chain stays single.- router = _router()- _isolate_branch(monkeypatch, router)- content = json.dumps(- [{"id": i, "status": "ok", "level": "INFO", "value": i * 2} for i in range(40)]- )- direct = _router()._get_smart_crusher().crush(content, query="q", bias=1.0).compressed- out, _tokens, chain = router._apply_strategy_to_content(- content, CompressionStrategy.SMART_CRUSHER, "q", bias=1.0- )- assert out == direct- assert chain == [CompressionStrategy.SMART_CRUSHER.value]---def _fallback_router() -> ContentRouter:- """Router with CODE_AWARE routing enabled and the if/elif branch terminal.-- Same isolation as :func:`_router` (no relevance split, no lossy layer, markers- off) but with ``enable_code_aware=True`` since that flag gates CODE_AWARE- ROUTING (default off) rather than the getter. HTML routing is on by default.- """- return ContentRouter(- ContentRouterConfig(- relevance_split=False,- lossless_then_lossy=False,- ccr_inject_marker=False,- enable_code_aware=True,- )- )---# ─────────────── flipped fallback strategies: CODE_AWARE / HTML ───────────────---def test_code_aware_router_dispatch_matches_direct(monkeypatch: pytest.MonkeyPatch) -> None:- # CODE_AWARE-SUCCEEDS: the flip routes through the registry "code_aware"- # adapter, which delegates to the SAME getter+method with language via config.- # A deterministic fake getter keeps this offline (no tree-sitter) and records- # the exact call args, proving content/language/context flow through unchanged.- router = _fallback_router()- _isolate_branch(monkeypatch, router)- seen: dict[str, object] = {}- shrunk = "def foo(): ... # compressed body"-- def _fake_compress(content: str, language: object = None, context: str = "") -> SimpleNamespace:- seen.update(content=content, language=language, context=context)- return SimpleNamespace(compressed=shrunk)-- monkeypatch.setattr(- router, "_get_code_compressor", lambda: SimpleNamespace(compress=_fake_compress)- )- # Sentinel ML: if the flip WRONGLY dropped into a Kompress fallback we'd see- # KOMPRESS appended to the chain; the chain assertion catches it.- monkeypatch.setattr(- router, "_try_ml_compressor", lambda *a, **k: ("KOMPRESS_SENTINEL", 999_999)- )-- content = "def foo():\n " + "x = 1\n " * 40 + "return x\n"- out, tokens, chain = router._apply_strategy_to_content(- content, CompressionStrategy.CODE_AWARE, "q", language="python", bias=1.0- )- assert out == shrunk- # Measured in the router's token estimate, the unit of original_tokens.- assert tokens == _estimate_tokens(shrunk)- assert chain == [CompressionStrategy.CODE_AWARE.value]- # The flip forwarded content, language, and context through the registry adapter.- assert seen == {"content": content, "language": "python", "context": "q"}---def test_code_aware_unavailable_falls_back_to_kompress(monkeypatch: pytest.MonkeyPatch) -> None:- # CODE_AWARE-RETURNS-NONE: with the code compressor UNAVAILABLE (tree-sitter- # missing) the branch's local `compressed` stays None, so the EXISTING inline- # Kompress fallback runs — the flip touches none of that logic. Mock- # _try_ml_compressor so NO real ML runs, and assert the SAME strategy_chain- # ([code_aware, kompress]) the historical direct dispatch produced.- router = _fallback_router()- _isolate_branch(monkeypatch, router)- monkeypatch.setattr(router, "_get_code_compressor", lambda: None)- monkeypatch.setattr(- router,- "_try_ml_compressor",- lambda content, ctx, q: ("KOMPRESSED::" + content, 3),- )- content = "def foo():\n return 1\n"- out, tokens, chain = router._apply_strategy_to_content(- content, CompressionStrategy.CODE_AWARE, "ctx", language=None, bias=1.0- )- assert out == "KOMPRESSED::" + content- assert tokens == 3- assert chain == [CompressionStrategy.CODE_AWARE.value, CompressionStrategy.KOMPRESS.value]---def test_html_router_dispatch_matches_direct(monkeypatch: pytest.MonkeyPatch) -> None:- # HTML-EXTRACT-SUCCEEDS: the flip routes through the registry "html" adapter,- # which delegates to the SAME getter + extract(). A deterministic fake getter- # keeps this offline (no trafilatura) and records the call arg.- router = _fallback_router()- _isolate_branch(monkeypatch, router)- extracted = "Extracted article body text that trafilatura would return."- seen: dict[str, object] = {}-- def _fake_extract(content: str) -> SimpleNamespace:- seen["content"] = content- return SimpleNamespace(extracted=extracted)-- monkeypatch.setattr(- router, "_get_html_extractor", lambda: SimpleNamespace(extract=_fake_extract)- )- content = "<html><body><article><p>hello world</p></article></body></html>"- out, tokens, chain = router._apply_strategy_to_content(- content, CompressionStrategy.HTML, "", bias=1.0- )- assert out == extracted- assert tokens == _estimate_tokens(extracted)- assert chain == [CompressionStrategy.HTML.value]- assert seen == {"content": content}--def test_html_extract_none_falls_through_to_passthrough(monkeypatch: pytest.MonkeyPatch) -> None:- # HTML-EXTRACT-NONE: when extraction yields None the adapter reports- # compressed=False, the branch's local `compressed` collapses to None, and the- # function falls through to the bottom passthrough exactly as the historical- # `result.extracted is None` path — chain [html, passthrough], content verbatim.- router = _fallback_router()- _isolate_branch(monkeypatch, router)+ router = ContentRouter(ContentRouterConfig(relevance_split=False, lossless_then_lossy=False))monkeypatch.setattr(router,"_get_html_extractor",lambda: SimpleNamespace(extract=lambda content: SimpleNamespace(extracted=None)),)- # Sentinel ML so we'd notice if the None path wrongly reached a lossy compressor.- monkeypatch.setattr(router, "_try_ml_compressor", lambda *a, **k: ("KOMPRESS_SENTINEL", 1))content = "<html><body><script>no extractable article body</script></body></html>"- out, tokens, chain = router._apply_strategy_to_content(- content, CompressionStrategy.HTML, "", bias=1.0+ out, _tokens, _chain, _reversible = router._apply_strategy_to_content(+ content, CompressionStrategy.HTML, "")assert out == content- assert tokens == _estimate_tokens(content)- assert chain == [CompressionStrategy.HTML.value, CompressionStrategy.PASSTHROUGH.value]---def test_diff_deferred_no_registry_entry() -> None:- # DIFF is DEFERRED: there is no "diff" built-in adapter/descriptor in the- # registry inventory, so registry resolution would return None and fall back- # to raw content — NOT byte-identical. It stays on its direct dispatch until a- # diff built-in adapter lands (PR-A scope).- router = _router()- assert router.compressor_registry.get("diff") is None- assert "diff" not in {d.name for d in _BUILTIN_COMPRESSOR_DESCRIPTORS}---def test_kompress_registry_dispatch_matches_direct(monkeypatch: pytest.MonkeyPatch) -> None:- # KOMPRESS now dispatches via the registry "kompress" adapter, which still- # delegates to _try_ml_compressor (the ML boundary). Mock the underlying model- # so no real ONNX/HF inference runs, and assert the compressed CONTENT is- # byte-identical to the direct _try_ml_compressor call (question is None here).- router = _router()- _isolate_branch(monkeypatch, router)- fake = SimpleNamespace(- is_ready=lambda: True,- ensure_background_load=lambda: None,- compress=lambda text, **kwargs: SimpleNamespace(- compressed="KOMPRESSED::" + text, compressed_tokens=7- ),- )- monkeypatch.setattr(router, "_get_kompress", lambda: fake)- content = "some plain text that the ML model would compress. " * 4- out, _tokens, chain = router._apply_strategy_to_content(- content, CompressionStrategy.KOMPRESS, "", bias=1.0- )- assert out == "KOMPRESSED::" + content- assert chain == [CompressionStrategy.KOMPRESS.value]- # CONTENT is byte-identical to the direct ML call (registry round-trip preserves- # content); the approved token-metric change is covered in- # test_router_registry_smartcrusher.py.- assert out == router._try_ml_compressor(content, "", None)[0]diff --git a/tests/test_router_registry_smartcrusher.py b/tests/test_router_registry_smartcrusher.pyindex d80684a31..8705bf516 100644--- a/tests/test_router_registry_smartcrusher.py+++ b/tests/test_router_registry_smartcrusher.py@@ -1,43 +1,8 @@-"""Differential tests for the SMART_CRUSHER / KOMPRESS / TEXT registry flip.--SMART_CRUSHER flips the PRIMARY compressor invocation to the compressor registry-(``_registry_compress("smart_crusher", ...)``), while the shared post-strategy-Kompress -> Log fallback block stays a direct dispatch. KOMPRESS and TEXT now ALSO-dispatch via the registry (``_registry_compress("kompress", ...)``): the-``kompress`` built-in adapter delegates to the SAME ``_try_ml_compressor(content,-context, question)`` the router historically called, with ``question`` forwarded-via ``CompressInput.config['question']`` (previously the adapter dropped it,-passing ``None`` — that latent bug is fixed here). Content is therefore PRESERVED-byte-for-byte against the direct ``_try_ml_compressor`` call.--The ONE approved, non-byte-identical change is the token metric: KOMPRESS/TEXT now-report ``_estimate_tokens(output.content)`` (the router's calibrated estimate)-instead of ``_try_ml_compressor``'s tuple token count (Kompress's own-``compressed_tokens``). No content/routing/fallback/lossless-then-lossy decision-reads that metric for these two branches, so only the reported number changes.--Every path asserts registry-dispatch CONTENT == the historical direct-dispatch-content (and, via recorded call args, that query/bias/question flow through), and-that the token metric equals ``_estimate_tokens(output.content)``.--Offline guardrails:- * No real ML/ONNX/HF inference — the ML boundary is mocked at- ``_try_ml_compressor`` (SMART_CRUSHER fallback tests) or ``_get_kompress``- (KOMPRESS/TEXT tests, which exercise the real ``_try_ml_compressor`` with a- fake in-memory model).- * SmartCrusher itself is pure-Python (no ML), so the SMART_CRUSHER success path- runs the real crusher on a small JSON array.- * ``relevance_split`` / ``lossless_then_lossy`` off and STAGE 0- (``_lossless_first``) neutralized so the if/elif branch under test is the- terminal path. All of that is shared, unchanged code.- * The broad ``content_router``/``compression`` -k selection is NOT exercised- (it hangs on HF-Hub/ONNX).-"""+"""Lossless JSON precedence and real no-saving fallback boundaries."""from __future__ import annotationsimport json-from types import SimpleNamespaceimport pytest@@ -49,255 +14,35 @@ from headroom.transforms.content_router import ()-def _router() -> ContentRouter:- """Router whose if/elif branch is the terminal dispatch path.-- ``relevance_split`` off, ``lossless_then_lossy`` off, and- ``ccr_inject_marker`` off make the compressed output deterministic and- marker-free; the same config is applied to the direct reference and the- dispatch router, so the differential holds regardless.- """- return ContentRouter([email protected]("pretty", [True, False], ids=["minification", "compact-fallback"])+def test_json_minification_precedes_fallback_without_losing_records(pretty: bool) -> None:+ records = [+ {+ "id": i,+ "status": "ok",+ "level": "ERROR" if i == 20 else "INFO",+ "message": f"CRITICAL_{i} transaction completed",+ }+ for i in range(60)+ ]+ content = (+ json.dumps(records, indent=2) if pretty else json.dumps(records, separators=(",", ":"))+ )+ router = ContentRouter(ContentRouterConfig(- relevance_split=False,- lossless_then_lossy=False,+ smart_crusher_with_compaction=False,+ smart_crusher_max_items_after_crush=1000,+ enable_kompress=False,ccr_inject_marker=False,))---def _isolate_branch(monkeypatch: pytest.MonkeyPatch, router: ContentRouter) -> None:- """Neutralize STAGE 0 (``_lossless_first``) so the if/elif branch is exercised.-- ``_lossless_first`` is shared, unchanged code (the flip only touches the branch- body), so forcing it to a no-op isolates what the flip actually changed.- """- monkeypatch.setattr(router, "_lossless_first", lambda content, strategy: (content, None))---# A JSON array the real SmartCrusher shrinks (so the fallback chain stays single).-_JSON = json.dumps([{"id": i, "status": "ok", "level": "INFO", "value": i * 2} for i in range(40)])---# ─────────────────────── SMART_CRUSHER: flipped (registry) ────────────────────---def test_smart_crusher_success_matches_direct(monkeypatch: pytest.MonkeyPatch) -> None:- # SMART_CRUSHER-SUCCEEDS: the flip routes the primary ``.crush`` through the- # registry "smart_crusher" adapter, which delegates to the SAME getter +- # ``.crush(query=..., bias=...)``. Differential: registry dispatch == a direct- # crush on an independent router. A real shrink means no fallback fires.- router = _router()- _isolate_branch(monkeypatch, router)- context, bias = "q", 1.0- direct = _router()._get_smart_crusher().crush(_JSON, query=context, bias=bias).compressed- out, tokens, chain = router._apply_strategy_to_content(- _JSON, CompressionStrategy.SMART_CRUSHER, context, bias=bias- )- assert out == direct- assert tokens == _estimate_tokens(direct)- assert chain == [CompressionStrategy.SMART_CRUSHER.value]- assert len(out) < len(_JSON)---def test_smart_crusher_forwards_query_and_bias(monkeypatch: pytest.MonkeyPatch) -> None:- # The flip must forward content, query (==context), and bias to ``.crush``- # unchanged. A fake crusher records the exact call args, proving the registry- # adapter's ``query=inp.query`` / ``bias=budget['bias']`` reproduce the direct- # ``crush(content, query=context, bias=bias)`` call.- router = _router()- _isolate_branch(monkeypatch, router)- seen: dict[str, object] = {}-- def _crush(content: str, query: str = "", bias: float = 1.0) -> SimpleNamespace:- seen.update(content=content, query=query, bias=bias)- return SimpleNamespace(compressed="CRUSHED " + content[:5])-- monkeypatch.setattr(router, "_get_smart_crusher", lambda: SimpleNamespace(crush=_crush))- # Sentinel ML: if the flip WRONGLY dropped into the Kompress fallback we'd see- # KOMPRESS appended to the chain; the chain assertion catches it.- monkeypatch.setattr(- router, "_try_ml_compressor", lambda *a, **k: ("KOMPRESS_SENTINEL", 999_999)- )-- content = "x" * 200- out, tokens, chain = router._apply_strategy_to_content(- content, CompressionStrategy.SMART_CRUSHER, "myquery", bias=0.7- )- assert out == "CRUSHED " + content[:5]- assert tokens == _estimate_tokens("CRUSHED " + content[:5])- assert chain == [CompressionStrategy.SMART_CRUSHER.value]- assert seen == {"content": content, "query": "myquery", "bias": 0.7}---def test_smart_crusher_kompress_fallback_matches_direct(monkeypatch: pytest.MonkeyPatch) -> None:- # SMART_CRUSHER-NO-SHRINK -> KOMPRESS: the crusher returns the content- # unchanged (no net saving), so the SHARED, UNCHANGED post-strategy block runs- # the Kompress fallback. The flip only sets ``compressed``/``compressed_tokens``- # on entry — identically to the direct crush — so the fallback fires exactly as- # before: chain [smart_crusher, kompress], Kompress output adopted.- router = _router()- _isolate_branch(monkeypatch, router)- monkeypatch.setattr(- router,- "_get_smart_crusher",- lambda: SimpleNamespace(crush=lambda c, query="", bias=1.0: SimpleNamespace(compressed=c)),+ compressed, tokens, chain, _reversible = router._apply_strategy_to_content(+ content, CompressionStrategy.SMART_CRUSHER, "CRITICAL_20")- # Kompress shrinks (fallback_tokens < compressed_tokens) so it is adopted.- # The fallback is measured with the router's estimator, not the count the- # ML compressor reports, so the mock must actually return less text.- kompressed = "KOMPRESSED::" + _JSON[:40]- monkeypatch.setattr(router, "_try_ml_compressor", lambda c, ctx, q: (kompressed, 3))- out, tokens, chain = router._apply_strategy_to_content(- _JSON, CompressionStrategy.SMART_CRUSHER, "ctx", bias=1.0- )- assert out == kompressed- assert tokens == _estimate_tokens(kompressed)- assert chain == [- CompressionStrategy.SMART_CRUSHER.value,- CompressionStrategy.KOMPRESS.value,- ]--def test_smart_crusher_log_fallback_matches_direct(monkeypatch: pytest.MonkeyPatch) -> None:- # SMART_CRUSHER-NO-SHRINK -> KOMPRESS-NO-SHRINK -> LOG: crusher passes content- # through, Kompress fails to shrink (fallback_tokens NOT < compressed_tokens),- # and — because the content is valid JSON and the log compressor is enabled —- # the last-ditch Log fallback runs and shrinks. All of that is the shared,- # unchanged block; the flip only feeds it identical entry state. Chain- # [smart_crusher, kompress, log].- router = _router()- _isolate_branch(monkeypatch, router)- monkeypatch.setattr(- router,- "_get_smart_crusher",- lambda: SimpleNamespace(crush=lambda c, query="", bias=1.0: SimpleNamespace(compressed=c)),- )- # Kompress returns content unchanged with a huge token count -> NOT a shrink,- # so the else branch (Log fallback) is taken.- monkeypatch.setattr(router, "_try_ml_compressor", lambda c, ctx, q: (c, 10**9))- monkeypatch.setattr(- router,- "_get_log_compressor",- lambda: SimpleNamespace(- compress=lambda c, bias=1.0: SimpleNamespace(compressed="LOG_FOLDED")- ),- )- out, tokens, chain = router._apply_strategy_to_content(- _JSON, CompressionStrategy.SMART_CRUSHER, "ctx", bias=1.0- )- assert out == "LOG_FOLDED"- assert tokens == _estimate_tokens("LOG_FOLDED")- assert chain == [- CompressionStrategy.SMART_CRUSHER.value,- CompressionStrategy.KOMPRESS.value,- CompressionStrategy.LOG.value,- ]---# ─────────────── KOMPRESS / TEXT: flipped (registry, ML boundary) ──────────────---def _fake_kompress(seen: dict[str, object]) -> SimpleNamespace:- """In-memory Kompress model: records compress kwargs, no ONNX/HF."""-- def _compress(text: str, **kwargs: object) -> SimpleNamespace:- seen.update(kwargs)- # ``compressed_tokens`` is the model's OWN count (7), deliberately unequal- # to ``_estimate_tokens`` of the output. After the flip the branch reports- # ``_estimate_tokens(output.content)`` and DISCARDS this tuple value — the- # single approved metric change — so the tests below assert the metric is- # the estimate, not 7.- return SimpleNamespace(compressed="KOMPRESSED::" + text, compressed_tokens=7)-- return SimpleNamespace(- is_ready=lambda: True,- ensure_background_load=lambda: None,- compress=_compress,- )---def test_kompress_registry_dispatch_matches_direct(monkeypatch: pytest.MonkeyPatch) -> None:- # KOMPRESS now dispatches through the registry "kompress" adapter, which- # delegates to _try_ml_compressor with ``question`` forwarded via config.- router = _router()- _isolate_branch(monkeypatch, router)- seen: dict[str, object] = {}- monkeypatch.setattr(router, "_get_kompress", lambda: _fake_kompress(seen))-- content = "some plain text that the ML model would compress. " * 4- out, tokens, chain = router._apply_strategy_to_content(- content, CompressionStrategy.KOMPRESS, "ctx", question="my question", bias=1.0- )- # (a) CONTENT preserved — byte-identical to the direct _try_ml_compressor call- # with the SAME question (so question is forwarded through the adapter).- assert out == "KOMPRESSED::" + content- assert chain == [CompressionStrategy.KOMPRESS.value]- direct, direct_tokens = router._try_ml_compressor(content, "ctx", "my question")- assert out == direct- # The real ``question`` reached the model (config['question'] -> adapter ->- # _try_ml_compressor -> compressor.compress), fixing the latent adapter bug- # that dropped it (passed None).- assert seen["question"] == "my question"- # (b) APPROVED metric change: the token count is now _estimate_tokens(output),- # NOT the model's own tuple value (7) that the direct call returns.- assert tokens == _estimate_tokens(out)- assert tokens != direct_tokens- assert direct_tokens == 7---def test_text_registry_dispatch_matches_direct(monkeypatch: pytest.MonkeyPatch) -> None:- # TEXT shares the "kompress" adapter — same guarantees as the KOMPRESS branch.- router = _router()- _isolate_branch(monkeypatch, router)- seen: dict[str, object] = {}- monkeypatch.setattr(router, "_get_kompress", lambda: _fake_kompress(seen))-- content = "plain prose the text strategy sends straight to kompress. " * 4- out, tokens, chain = router._apply_strategy_to_content(- content, CompressionStrategy.TEXT, "ctx", question="q2", bias=1.0- )- assert out == "KOMPRESSED::" + content- assert chain == [CompressionStrategy.TEXT.value]- direct, direct_tokens = router._try_ml_compressor(content, "ctx", "q2")- assert out == direct- assert seen["question"] == "q2"- assert tokens == _estimate_tokens(out)- assert tokens != direct_tokens- assert direct_tokens == 7---def test_kompress_question_forwarding_changes_content(monkeypatch: pytest.MonkeyPatch) -> None:- # QA-differential: a question-aware fake model embeds the question in its- # output, so a DIFFERENT ``question`` yields DIFFERENT content. Proves the- # router forwards ``question`` end-to-end via the registry adapter- # (config['question'] -> _invoke_kompress -> _try_ml_compressor ->- # compressor.compress) — the fix for the adapter that previously dropped it.- router = _router()- _isolate_branch(monkeypatch, router)-- def _qa_model() -> SimpleNamespace:- def _compress(text: str, *, question: object = None, **kwargs: object) -> SimpleNamespace:- return SimpleNamespace(compressed=f"Q[{question}]::{text}", compressed_tokens=5)-- return SimpleNamespace(- is_ready=lambda: True,- ensure_background_load=lambda: None,- compress=_compress,- )-- monkeypatch.setattr(router, "_get_kompress", _qa_model)- content = "the body the model compresses conditioned on the question. " * 3-- out_a, _tok_a, _ = router._apply_strategy_to_content(- content, CompressionStrategy.KOMPRESS, "ctx", question="alpha", bias=1.0- )- out_b, _tok_b, _ = router._apply_strategy_to_content(- content, CompressionStrategy.KOMPRESS, "ctx", question="beta", bias=1.0- )- assert out_a != out_b- assert out_a.startswith("Q[alpha]::")- assert out_b.startswith("Q[beta]::")- # Each matches the direct _try_ml_compressor call with the SAME question.- assert out_a == router._try_ml_compressor(content, "ctx", "alpha")[0]- assert out_b == router._try_ml_compressor(content, "ctx", "beta")[0]+ assert json.loads(compressed) == records+ if pretty:+ assert chain == [CompressionStrategy.JSON_MINIFIER.value]+ assert tokens < _estimate_tokens(content)+ else:+ assert compressed == contentdiff --git a/tests/test_stateless_no_disk.py b/tests/test_stateless_no_disk.pyindex 7b773a197..4092f60ac 100644--- a/tests/test_stateless_no_disk.py+++ b/tests/test_stateless_no_disk.py@@ -30,6 +30,7 @@ from fastapi.testclient import TestClientfrom headroom import pathsfrom headroom.cache import compression_store as csfrom headroom.cache.backends import InMemoryBackend+from headroom.parser import CCR_RETRIEVAL_MARKER_RE# The runtime log is the one file stateless mode is documented to keep writing# (it is created owner-only and carries no request content by default).@@ -82,15 +83,14 @@ def _snapshot(root: Path) -> set[str]:def _tool_heavy_messages() -> list[dict]:large = json.dumps([- {- "id": i,- "name": f"Item {i}",- "description": (- f"Detailed description for item {i}: status active, created on "- f"2024-01-{(i % 28) + 1:02d}, category=electronics, price={i * 10.99:.2f}."- ),- "tags": ["electronics", "sale", "featured"],- }+ # Text rows cannot take table compaction's lossless no-cache path.+ # Repetition gives the real keep/drop transform and CCR something+ # to exercise without relying on serialization-only token savings.+ (+ f"Detailed description for item {i % 20}: status active, created on "+ f"2024-01-{(i % 20) + 1:02d}, category=electronics, "+ f"price={(i % 20) * 10.99:.2f}. Tags: electronics, sale, featured."+ )for i in range(200)])@@ -123,12 +123,31 @@ def _run_turn(stateless: bool) -> None:cost_tracking_enabled=False,)app = create_app(config)+ messages = _tool_heavy_messages()with TestClient(app, base_url="http://127.0.0.1", client=("127.0.0.1", 12345)) as client:+ # The default sidecar mode deliberately disables CCR store writes.resp = client.post(- "/v1/compress", json={"messages": _tool_heavy_messages(), "model": "gpt-4"}+ "/v1/compress",+ json={"messages": messages, "model": "gpt-4", "config": {"mode": "ccr"}},)assert resp.status_code == 200, resp.text- assert resp.json()["tokens_saved"] > 0, "turn must actually exercise the CCR store"+ data = resp.json()+ assert data["tokens_saved"] > 0, "turn must actually compress content"+ original = json.loads(messages[2]["content"])+ compressed, _ = json.JSONDecoder().raw_decode(data["messages"][2]["content"])+ kept = [+ row+ for row in compressed+ if isinstance(row, str) and not CCR_RETRIEVAL_MARKER_RE.search(row)+ ]+ assert len(kept) < len(original)+ assert all(row in original for row in kept)+ retrieved = [client.post("/v1/retrieve", json={"hash": key}) for key in data["ccr_hashes"]]+ for response in retrieved:+ assert response.status_code == 200, response.text+ assert any(+ json.loads(response.json()["original_content"]) == original for response in retrieved+ ), "the emitted CCR hash must recover every original tool row"# ---- end to end -----------------------------------------------------------diff --git a/tests/test_transforms/test_content_router.py b/tests/test_transforms/test_content_router.pyindex d714b1224..3733d3fdc 100644--- a/tests/test_transforms/test_content_router.py+++ b/tests/test_transforms/test_content_router.py@@ -1463,10 +1463,12 @@ class TestSmartCrusherFallback:),)- compressed, compressed_tokens, strategy_chain = router._apply_strategy_to_content(- content,- CompressionStrategy.SMART_CRUSHER,- context="",+ compressed, compressed_tokens, strategy_chain, _reversible = (+ router._apply_strategy_to_content(+ content,+ CompressionStrategy.SMART_CRUSHER,+ context="",+ ))# Strategy chain must include smart_crusher@@ -1508,10 +1510,12 @@ class TestSmartCrusherFallback:lambda self: mock_crusher,)- compressed, compressed_tokens, strategy_chain = router._apply_strategy_to_content(- content,- CompressionStrategy.SMART_CRUSHER,- context="",+ compressed, compressed_tokens, strategy_chain, _reversible = (+ router._apply_strategy_to_content(+ content,+ CompressionStrategy.SMART_CRUSHER,+ context="",+ ))# SmartCrusher should handle JSON directly@@ -1566,10 +1570,12 @@ class TestSmartCrusherFallback:),)- compressed, compressed_tokens, strategy_chain = router._apply_strategy_to_content(- repetitive,- CompressionStrategy.SMART_CRUSHER,- context="",+ compressed, compressed_tokens, strategy_chain, _reversible = (+ router._apply_strategy_to_content(+ repetitive,+ CompressionStrategy.SMART_CRUSHER,+ context="",+ ))# The chain must include the requested strategy@@ -1599,10 +1605,12 @@ class TestSmartCrusherFallback:plain = "This is just plain text. " * 200- compressed, compressed_tokens, strategy_chain = router._apply_strategy_to_content(- plain,- CompressionStrategy.CODE_AWARE,- context="",+ compressed, compressed_tokens, strategy_chain, _reversible = (+ router._apply_strategy_to_content(+ plain,+ CompressionStrategy.CODE_AWARE,+ context="",+ ))# CodeAware should be in the chain@@ -1829,7 +1837,7 @@ class TestMixedContentTagProtection:router = ContentRouter(ContentRouterConfig(min_section_tokens=1))def mangle(content, strategy, context, language=None, question=None, bias=1.0):- return "MANGLED", 1, None+ return "MANGLED", 1, [strategy.value], Falserouter._apply_strategy_to_content = mangle # type: ignore[method-assign]return router@@ -1865,3 +1873,23 @@ class TestMixedContentTagProtection:assert "MANGLED" in result.compressedassert result.strategy_used == CompressionStrategy.MIXED+++class TestMixedContentAttribution:+ def test_minified_json_with_unchanged_code_remains_mixed(self):+ value = {f"field_{i}": i for i in range(500)}+ code = "print('preserve this unchanged')"+ content = f"```python\n{code}\n```\n\n{json.dumps(value, indent=2)}"+ router = ContentRouter(ContentRouterConfig(min_section_tokens=10, enable_kompress=False))++ result = router._compress_mixed(content, context="")++ assert result.strategy_used == CompressionStrategy.MIXED+ assert code in result.compressed+ json_block = result.compressed.split("```", 2)[2].strip()+ assert json.loads(json_block) == value+ assert {decision.strategy for decision in result.routing_log} == {+ CompressionStrategy.CODE_AWARE,+ CompressionStrategy.JSON_MINIFIER,+ }+ assert result.total_compressed_tokens < result.total_original_tokensdiff --git a/tests/test_transforms/test_smart_crusher_rust_parity.py b/tests/test_transforms/test_smart_crusher_rust_parity.pyindex 79ccecebe..adb4fe93d 100644--- a/tests/test_transforms/test_smart_crusher_rust_parity.py+++ b/tests/test_transforms/test_smart_crusher_rust_parity.py@@ -42,16 +42,32 @@ _FIXTURES_DIR = Path(__file__).parent.parent / "parity" / "fixtures" / "smart_crdef _all_fixtures() -> list[Path]:- return sorted(_FIXTURES_DIR.glob("*.json"))---def test_at_least_17_fixtures_present():- """Sanity check: the recorded fixture suite landed."""- fixtures = _all_fixtures()- assert len(fixtures) >= 17, (- f"expected >= 17 fixtures, found {len(fixtures)}. "- "If you re-recorded and got fewer, something deleted them."- )+ fixtures = sorted(_FIXTURES_DIR.glob("*.json"))+ labels = {p.stem.rsplit("_", 1)[0] for p in fixtures}+ # Require each native boundary and transformed-value scenario in the matrix;+ # an arbitrary fixture count can hide a deleted scenario behind duplicates.+ for label in (+ "non_json_passthrough",+ "dict_array_30",+ "string_array_25",+ "number_array_40_changepoint",+ "mixed_array",+ "dict_array_30_bias_high",+ "dict_array_30_bias_low",+ "dict_array_100_sequential",+ "duplicate_dicts_40",+ "empty_array",+ "small_object_passthrough",+ "short_array_passthrough",+ "nulls_and_bools",+ "unicode_dict_array",+ "time_series_50",+ "nested_object_with_array",+ "nested_3deep_with_array",+ ):+ if label not in labels:+ raise FileNotFoundError(f"Missing native parity scenario: {label}")+ return fixtures@pytest.mark.parametrize("fixture_path", _all_fixtures(), ids=lambda p: p.name)diff --git a/tests/test_transforms_config_compressor.py b/tests/test_transforms_config_compressor.pyindex 96d778c6a..56a30d566 100644--- a/tests/test_transforms_config_compressor.py+++ b/tests/test_transforms_config_compressor.py@@ -572,19 +572,9 @@ def test_router_compresses_k8s_manifest_end_to_end() -> None:def test_router_lossless_mode_uses_lossless_config_label() -> None:router = ContentRouter(ContentRouterConfig(lossless=True))- compressed, _tokens, chain = router._apply_strategy_to_content(+ compressed, _tokens, chain, _reversible = router._apply_strategy_to_content(K8S_MANIFEST, CompressionStrategy.CONFIG, context="")assert chain == ["lossless_config"]assert expand_runs(unfold_repeated_blocks(compressed)) == K8S_MANIFESTassert not CCR_RETRIEVAL_MARKER_RE.search(compressed)---def test_router_disabled_flag_skips_config_compressor() -> None:- router = ContentRouter(ContentRouterConfig(enable_config_compressor=False))- compressed, _tokens, _chain = router._apply_strategy_to_content(- K8S_MANIFEST, CompressionStrategy.CONFIG, context=""- )- # Falls through to the unified fallback path; must not crash and must- # not claim CONFIG did work it didn't do.- assert isinstance(compressed, str)diff --git a/tests/test_transforms_content_router.py b/tests/test_transforms_content_router.pyindex 6c37a5a9c..3c1909dae 100644--- a/tests/test_transforms_content_router.py+++ b/tests/test_transforms_content_router.py@@ -512,96 +512,6 @@ def test_force_kompress_apply_lightweight_detection_protects_recent_code(assert result.transforms_applied == ["router:protected:recent_code"]-def test_content_router_mixed_pure_apply_and_toin(monkeypatch: pytest.MonkeyPatch) -> None:- router = ContentRouter()- mixed_content = "\n".join(["before", "```python", "print('x')", "```", "after"])- monkeypatch.setattr(- content_router_module,- "split_into_sections",- lambda content, isolate=(): [- SimpleNamespace(- content="print('x')",- content_type=ContentType.SOURCE_CODE,- language="python",- is_code_fence=True,- ),- SimpleNamespace(- content="after text",- content_type=ContentType.PLAIN_TEXT,- language=None,- is_code_fence=False,- ),- ],- )- monkeypatch.setattr(- router,- "_apply_strategy_to_content",- lambda content, strategy, context, language=None, question=None, bias=1.0: (- f"{strategy.value}:{content}",- len(content.split()) - 1,- [strategy.value],- ),- )- result = router._compress_mixed(mixed_content, "ctx")- assert result.strategy_used is CompressionStrategy.MIXED- assert result.sections_processed == 2- assert "```python\ncode_aware:print('x')\n```" in result.compressed-- monkeypatch.setattr(- router,- "_apply_strategy_to_content",- lambda content, strategy, context, language=None, question=None, bias=1.0: (- "shrunk",- 1,- [strategy.value],- ),- )- pure = router._compress_pure("some plain text", CompressionStrategy.TEXT, "ctx")- assert pure.routing_log[0].content_type is ContentType.PLAIN_TEXT- assert pure.total_original_tokens == _estimate_tokens("some plain text")- assert pure.total_compressed_tokens == 1-- calls: list[dict] = []- router._toin = SimpleNamespace(record_compression=lambda **kwargs: calls.append(kwargs))- monkeypatch.setattr(content_router_module, "_create_content_signature", lambda **kwargs: "sig")- router._record_to_toin(- CompressionStrategy.TEXT,- "original content",- "small",- original_tokens=10,- compressed_tokens=4,- language="python",- context="question",- )- assert calls[0]["tool_signature"] == "sig"- assert calls[0]["strategy"] == "text"- assert calls[0]["query_context"] == "question"-- router._record_to_toin(- CompressionStrategy.SMART_CRUSHER,- "x",- "x",- original_tokens=10,- compressed_tokens=4,- )- router._record_to_toin(- CompressionStrategy.TEXT,- "x",- "x",- original_tokens=2,- compressed_tokens=2,- )- monkeypatch.setattr(content_router_module, "_create_content_signature", lambda **kwargs: None)- router._record_to_toin(- CompressionStrategy.TEXT,- "x",- "y",- original_tokens=5,- compressed_tokens=1,- )- assert len(calls) == 1--def test_diff_strategy_does_not_fallback_to_kompress_when_diff_is_noop(monkeypatch: pytest.MonkeyPatch,) -> None:@@ -619,7 +529,7 @@ def test_diff_strategy_does_not_fallback_to_kompress_when_diff_is_noop(monkeypatch.setattr(router, "_try_ml_compressor", fail_kompress)- compressed, compressed_tokens, strategy_chain = router._apply_strategy_to_content(+ compressed, compressed_tokens, strategy_chain, _reversible = router._apply_strategy_to_content(diff,CompressionStrategy.DIFF,context="",@@ -647,7 +557,7 @@ def test_log_strategy_does_not_fallback_to_kompress_when_log_is_noop(monkeypatch.setattr(router, "_try_ml_compressor", fail_kompress)- compressed, compressed_tokens, strategy_chain = router._apply_strategy_to_content(+ compressed, compressed_tokens, strategy_chain, _reversible = router._apply_strategy_to_content(log,CompressionStrategy.LOG,context="",@@ -693,7 +603,7 @@ def test_smart_crusher_log_fallback_skipped_for_invalid_json(lambda content, context, question=None: (content, len(content.split())),)- compressed, compressed_tokens, strategy_chain = router._apply_strategy_to_content(+ compressed, compressed_tokens, strategy_chain, _reversible = router._apply_strategy_to_content(truncated,CompressionStrategy.SMART_CRUSHER,context="",@@ -1077,49 +987,6 @@ def test_relevance_split_does_not_fragment_json_payloads(assert kompress_inputs == []-def test_smart_crusher_log_fallback_runs_for_valid_json(- monkeypatch: pytest.MonkeyPatch,-) -> None:- """Valid JSON arrays still reach the Log fallback when SmartCrusher no-ops.-- The JSON-validity guard (#1306) only blocks the Log fallback for- *invalid* JSON. Valid JSON arrays that SmartCrusher can't shrink (e.g.- repetitive JSONL the proxy's own log dumps) still get the Log fallback,- which is the intended "repetitive JSONL" path.- """- router = ContentRouter(ContentRouterConfig())- valid_json = '[{"a": 1}, {"a": 1}, {"a": 1}, {"a": 1}]'-- class NoopSmartCrusher:- def crush(self, content: str, query: str = "", bias: float = 1.0) -> SimpleNamespace:- return SimpleNamespace(compressed=content)-- class ShrinkingLogCompressor:- def compress(self, content: str, bias: float = 1.0) -> SimpleNamespace:- return SimpleNamespace(compressed="[compressed]")-- monkeypatch.setattr(router, "_get_smart_crusher", lambda: NoopSmartCrusher())- monkeypatch.setattr(router, "_get_log_compressor", lambda: ShrinkingLogCompressor())- # Kompress no-op → Log fallback fires. A faithful no-op reports the same- # token count the router computed for the (unchanged) content — using- # _estimate_tokens, not a naive word split, so it isn't mistaken for a- # saving once #1857's whitespace-aware counting rates the JSON above 8.- monkeypatch.setattr(- router,- "_try_ml_compressor",- lambda content, context, question=None: (content, _estimate_tokens(content)),- )-- compressed, _compressed_tokens, strategy_chain = router._apply_strategy_to_content(- valid_json,- CompressionStrategy.SMART_CRUSHER,- context="",- )-- assert CompressionStrategy.LOG.value in strategy_chain- assert compressed == "[compressed]"--# ---------------------------------------------------------------------------# MIXED false-positive on source code: Python with dict literals + docstrings# triggers ``has_json_blocks`` and ``has_prose`` in ``is_mixed_content``,@@ -2633,41 +2500,34 @@ def test_html_with_embedded_json_runs_extractor_not_json_shortcut() -> None:assert len(result.compressed) < len(doc) * 3 // 5-def test_html_extraction_miss_still_gets_deferred_json_splice() -> None:- """Deferring must not LOSE the embedded-JSON win when extraction finds nothing.-- A shell-only HTML document makes the extractor come back empty; the- deferred splice then still compresses the embedded array. Before the fix- this shape short-circuited at the top (chain ``["embedded_json"]``); now- the block first proves the HTML strategy is a miss (chain- ``["html", "embedded_json"]``) with the same spliced output.- """- pytest.importorskip("trafilatura")- embedded = json.dumps(- [- {"id": i, "name": f"item-{i}", "tags": ["alpha", "beta"], "score": i * 3}- for i in range(40)- ]+def test_json_in_html_preserves_records_and_markup() -> None:+ rows = [+ {"id": i, "account": f"ACCOUNT_{i}", "score": i * 13, "active": i % 2 == 0}+ for i in range(60)+ ]+ prefix = (+ "<!DOCTYPE html>\n<html><head><title>data</title></head>\n<body>\n"+ '<script type="application/json">')- shell = (- "<!DOCTYPE html>\n<html><head><title>t</title></head>\n<body>\n"- '<script type="application/json">' + embedded + "</script>\n"- "</body></html>\n"+ suffix = "</script>\n</body></html>\n"+ shell = prefix + json.dumps(rows) + suffix+ router = ContentRouter(+ ContentRouterConfig(+ lossless=False,+ enable_html_extractor=True,+ smart_crusher_lossless_only=True,+ smart_crusher_with_compaction=False,+ enable_kompress=False,+ ))- router = ContentRouter()- # sanity: extraction finds nothing on this shell (empty or None)- extractor = router._get_html_extractor()- assert extractor is not None- assert not (extractor.extract(shell).extracted or "").strip()- text, _tokens, chain = router._apply_strategy_to_content(- shell, CompressionStrategy.HTML, "", None, 1.0- )+ result = router.compress(shell)- assert chain == ["html", "embedded_json"]- assert len(text) < len(shell)- json_marker = '"id":'- assert json_marker not in text # the array was compressed away+ assert result.compressed.startswith(prefix)+ assert result.compressed.endswith(suffix)+ assert json.loads(result.compressed[len(prefix) : -len(suffix)]) == rows+ assert result.tokens_saved > 0+ assert not result.unrecoverable_sections# A Claude Code Read of Rust: line-numbered code that code_aware hands back@@ -2713,7 +2573,7 @@ def test_code_aware_noop_keeps_a_smaller_kompress_fallback(assert _estimate_tokens(kompressed) > len(_NUMBERED_RUST.split())router = _noop_code_aware_router(monkeypatch, kompressed)- compressed, compressed_tokens, strategy_chain = router._apply_strategy_to_content(+ compressed, compressed_tokens, strategy_chain, _reversible = router._apply_strategy_to_content(_NUMBERED_RUST, CompressionStrategy.CODE_AWARE, context="")@@ -2728,7 +2588,7 @@ def test_code_aware_noop_reports_tokens_when_kompress_passes_through("""An unchanged block is reported at its own token count, never its word count."""router = _noop_code_aware_router(monkeypatch, _NUMBERED_RUST)- compressed, compressed_tokens, _chain = router._apply_strategy_to_content(+ compressed, compressed_tokens, _chain, _reversible = router._apply_strategy_to_content(_NUMBERED_RUST, CompressionStrategy.CODE_AWARE, context="")@@ -2751,7 +2611,7 @@ def test_code_aware_noop_runs_kompress_once_when_it_passes_through(monkeypatch.setattr(router, "_try_ml_compressor", counting)- compressed, compressed_tokens, _chain = router._apply_strategy_to_content(+ compressed, compressed_tokens, _chain, _reversible = router._apply_strategy_to_content(_NUMBERED_RUST, CompressionStrategy.CODE_AWARE, context="")@@ -2780,7 +2640,7 @@ def test_code_aware_noop_keeps_a_smaller_inline_kompress(monkeypatch.setattr(router, "_try_ml_compressor", counting)- compressed, compressed_tokens, strategy_chain = router._apply_strategy_to_content(+ compressed, compressed_tokens, strategy_chain, _reversible = router._apply_strategy_to_content(_NUMBERED_RUST, CompressionStrategy.CODE_AWARE, context="")@@ -2788,3 +2648,138 @@ def test_code_aware_noop_keeps_a_smaller_inline_kompress(assert compressed_tokens == _estimate_tokens(kompressed)assert strategy_chain == ["code_aware", "kompress"]assert len(calls) == 1++[email protected]("as_block", [False, True], ids=["string-tool", "tool-result-block"])[email protected]("other_marker", [False, True], ids=["no-marker", "unrelated-marker"])[email protected]("ccr_enabled", [False, True], ids=["explicit-no-ccr", "reversible-tools"])+def test_mixed_lossy_section_cannot_borrow_another_sections_retrieval_marker(+ monkeypatch, as_block, other_marker, ccr_enabled+) -> None:+ router = ContentRouter(+ ContentRouterConfig(+ enable_code_aware=True,+ enable_kompress=False,+ min_chars_for_block_compression=0,+ protect_recent_code=0,+ protect_analysis_context=False,+ ccr_inject_marker=ccr_enabled,+ )+ )+ code = (+ "def account():\n"+ + "".join(+ f" # audit record {i}: preserve the complete account history.\n" for i in range(120)+ )+ + ' identifier = "ACCOUNT_ID_42"\n return identifier\n'+ )+ content = "```python\n" + code + "```\n\n" + '[{"flag":true,"value":7}]'+ if other_marker:+ content += "\n\nIndependent retained section <<ccr:123456789012>>"++ def compress_section(section, strategy, *_args, **_kwargs):+ # Control only the lossy compressor boundary; split/routing/guards are real.+ compressed = (+ section.replace("ACCOUNT_ID_42", "")+ if strategy is CompressionStrategy.CODE_AWARE+ else section+ )+ return compressed, _estimate_tokens(compressed), [strategy.value], compressed == section++ monkeypatch.setattr(router, "_apply_strategy_to_content", compress_section)+ monkeypatch.setattr(+ router,+ "compress",+ lambda c, context="", bias=1.0, precomputed_detection=None: router._compress_mixed(+ c, context, bias=bias+ ),+ )+ direct = router.compress(content)+ assert direct.strategy_used is CompressionStrategy.MIXED+ assert direct.compression_ratio < 1.0+ assert "ACCOUNT_ID_42" not in direct.compressed++ message = (+ _tool_result_block_msg(content)+ if as_block+ else {"role": "tool", "tool_call_id": "account", "content": content}+ )+ for _turn in range(2):+ result = router.apply([message], _ChurnTokenizer(), model_limit=100_000)+ output = (+ result.messages[0]["content"][0]["content"]+ if as_block+ else result.messages[0]["content"]+ )+ if ccr_enabled or other_marker:+ assert output == content+ else:+ assert "ACCOUNT_ID_42" not in output+ assert '[{"flag":true,"value":7}]' in output++[email protected]("as_block", [False, True], ids=["string-tool", "tool-result-block"])[email protected]("mode", ["minify", "unmarked-loss", "sibling-marker"])+def test_embedded_json_mixed_tool_output_preserves_every_record(+ monkeypatch: pytest.MonkeyPatch, as_block: bool, mode: str+) -> None:+ from headroom.providers import OpenAIProvider+ from headroom.tokenizer import Tokenizer++ monkeypatch.setenv("HEADROOM_DETECT_BACKEND", "python")+ router = ContentRouter(+ ContentRouterConfig(+ smart_crusher_with_compaction=False,+ smart_crusher_max_items_after_crush=1000,+ enable_kompress=False,+ lossless_then_lossy=False,+ relevance_split=False,+ min_chars_for_block_compression=0,+ protect_recent_code=0,+ protect_analysis_context=False,+ ccr_inject_marker=True,+ )+ )+ rows = [{"id": i, "account": f"ACCOUNT_ID_{i}", "value": i * 13} for i in range(60)]+ fenced = "\n".join(f"Retain the complete diagnostic note for customer {i}." for i in range(30))+ content = f"```text\n{fenced}\n```\n\n"+ if mode == "sibling-marker":+ sibling = json.dumps([{"recover": i} for i in range(20)])+ content += f"Independent: {sibling}\n\n"+ content += f"Records: {json.dumps(rows)}\n\nDone."+ if mode != "minify":+ # Control the lossy compressor only. Nested dispatch, span adoption,+ # mixed routing, both safety guards and their caches remain production code.+ def crush(span, **_kwargs):+ values = json.loads(span)+ text = (+ "<<ccr:123456789012>>"+ if "recover" in values[0]+ else json.dumps(values[:1], separators=(",", ":"))+ )+ return SimpleNamespace(compressed=text)++ monkeypatch.setattr(router, "_get_smart_crusher", lambda: SimpleNamespace(crush=crush))++ tokenizer = Tokenizer(OpenAIProvider().get_token_counter("gpt-4o"), "gpt-4o")+ message = (+ _tool_result_block_msg(content)+ if as_block+ else {"role": "tool", "tool_call_id": "records", "content": content}+ )+ for _turn in range(2): # Cold compression followed by the cached/skip verdict.+ result = router.apply([message], tokenizer, model_limit=100_000)+ output = (+ result.messages[0]["content"][0]["content"]+ if as_block+ else result.messages[0]["content"]+ )+ if mode == "minify":+ assert tokenizer.count_text(output) < tokenizer.count_text(content)+ assert f"```text\n{fenced}\n```" in output+ assert output.endswith("Done.")+ assert "<<ccr:" not in output+ recovered, _end = json.JSONDecoder().raw_decode(output.split("Records: ", 1)[1])+ assert recovered == rows+ else:+ assert output == content