Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(smart-crusher): preserve unchanged JSON bytes (#4044)

Mansurshakh Japarov93279f3208b1
diff --git a/crates/headroom-core/src/transforms/live_zone.rs b/crates/headroom-core/src/transforms/live_zone.rs
index a41fee3d2..72d016fe2 100644
--- a/crates/headroom-core/src/transforms/live_zone.rs
+++ b/crates/headroom-core/src/transforms/live_zone.rs
@@ -115,6 +115,7 @@ use super::diff_compressor::{DiffCompressor, DiffCompressorConfig};
#[cfg(feature = "ml")]
use super::kompress::{Kompress, KompressConfig};
use super::log_compressor::{LogCompressor, LogCompressorConfig};
+use super::pipeline::JsonMinifier;
use super::search_compressor::{SearchCompressor, SearchCompressorConfig};
use super::smart_crusher::{SmartCrusher, SmartCrusherConfig};
use crate::ccr::{compute_key, marker_for, CcrStore};
@@ -1176,8 +1177,11 @@ fn compress_one_block(
// stays honest — the marker costs ~6 tokens and we'd
// rather forward the original than ship a bigger payload
// for a 5-byte block.
- let (compressed_for_replacement, ccr_hash_emitted) =
- maybe_inject_ccr_marker(content_text, &compressed, ccr_store);
+ let (compressed_for_replacement, ccr_hash_emitted) = if strategy == "json_minifier" {
+ (compressed, None)
+ } else {
+ maybe_inject_ccr_marker(content_text, &compressed, ccr_store)
+ };
let compressed_bytes = compressed_for_replacement.len();
// 3. Tokenizer-validated rejection. Per PR-B4 spec we
// count both the original and compressed strings
@@ -1659,6 +1663,12 @@ fn dispatch_compressor(text: &str, content_type: ContentType) -> DispatchResult
// arrays, and returns the input.
let result = smart_crusher().crush(text, EMPTY_QUERY, DEFAULT_BIAS);
if !result.was_modified {
+ if let Ok(Some(minified)) = JsonMinifier.minify(text) {
+ return DispatchResult::Compressed {
+ strategy: "json_minifier",
+ compressed: minified,
+ };
+ }
return DispatchResult::NoOp {
content_type: content_type.as_str(),
};
diff --git a/crates/headroom-core/src/transforms/pipeline/reformats/json_minifier.rs b/crates/headroom-core/src/transforms/pipeline/reformats/json_minifier.rs
index 8989add80..1df8e61c0 100644
--- a/crates/headroom-core/src/transforms/pipeline/reformats/json_minifier.rs
+++ b/crates/headroom-core/src/transforms/pipeline/reformats/json_minifier.rs
@@ -1,9 +1,8 @@
//! `JsonMinifier` — a [`ReformatTransform`] that strips insignificant
-//! whitespace from JSON via `serde_json` round-trip.
+//! whitespace from validated JSON without reserializing its values.
//!
-//! Lossless by construction: parse to `serde_json::Value`, re-emit
-//! compactly, return whichever is shorter. No CCR involvement —
-//! reformat transforms never need it.
+//! Strings, escapes, number literals, key order and duplicate keys remain
+//! byte-identical. No CCR involvement — reformat transforms never need it.
//!
//! [`ReformatTransform`]: crate::transforms::pipeline::traits::ReformatTransform
@@ -17,6 +16,43 @@ const NAME: &str = "json_minifier";
#[derive(Debug, Default, Clone, Copy)]
pub struct JsonMinifier;
+impl JsonMinifier {
+ /// Return a smaller data-equivalent JSON representation, or no rewrite.
+ pub fn minify(&self, content: &str) -> Result<Option<String>, TransformError> {
+ if content.trim().is_empty() {
+ return Err(TransformError::skipped(NAME, "empty input"));
+ }
+ let _: &serde_json::value::RawValue = serde_json::from_str(content)
+ .map_err(|e| TransformError::invalid_input(NAME, e.to_string()))?;
+ let mut output: Option<String> = None;
+ let mut segment_start = 0;
+ let mut quoted = false;
+ let mut escaped = false;
+ for (index, byte) in content.bytes().enumerate() {
+ if quoted {
+ if escaped {
+ escaped = false;
+ } else if byte == b'\\' {
+ escaped = true;
+ } else if byte == b'"' {
+ quoted = false;
+ }
+ } else if byte == b'"' {
+ quoted = true;
+ } else if matches!(byte, b' ' | b'\n' | b'\r' | b'\t') {
+ output
+ .get_or_insert_with(|| String::with_capacity(content.len() - 1))
+ .push_str(&content[segment_start..index]);
+ segment_start = index + 1;
+ }
+ }
+ if let Some(output) = &mut output {
+ output.push_str(&content[segment_start..]);
+ }
+ Ok(output)
+ }
+}
+
impl ReformatTransform for JsonMinifier {
fn name(&self) -> &'static str {
NAME
@@ -30,40 +66,25 @@ impl ReformatTransform for JsonMinifier {
}
fn apply(&self, content: &str) -> Result<ReformatOutput, TransformError> {
- let trimmed = content.trim();
- if trimmed.is_empty() {
- return Err(TransformError::skipped(NAME, "empty input"));
- }
-
- let value: serde_json::Value = serde_json::from_str(trimmed)
- .map_err(|e| TransformError::invalid_input(NAME, e.to_string()))?;
-
- let minified = serde_json::to_string(&value)
- .map_err(|e| TransformError::internal(NAME, e.to_string()))?;
-
- // Defensive: if minification grew the byte count (e.g. caller
- // already passed compact JSON, or escaping rules added bytes),
- // hand the original back so we never inflate the wire output.
- if minified.len() >= content.len() {
- return Ok(ReformatOutput::from_lengths(
- content.len(),
- content.to_string(),
- ));
- }
-
- Ok(ReformatOutput::from_lengths(content.len(), minified))
+ let output = self.minify(content)?.unwrap_or_else(|| content.to_string());
+ Ok(ReformatOutput::from_lengths(content.len(), output))
}
}
#[cfg(test)]
mod tests {
use super::*;
-
#[test]
- fn name_and_applies_to() {
- let m = JsonMinifier;
- assert_eq!(m.name(), "json_minifier");
- assert_eq!(m.applies_to(), &[ContentType::JsonArray]);
+ fn minification_preserves_json_literals() {
+ let input =
+ r#" { "z": 123456789012345678901234567890, "a": 1.2300e+400, "z": "\u0061 \" b 🌍" } "#;
+ let compact =
+ r#"{"z":123456789012345678901234567890,"a":1.2300e+400,"z":"\u0061 \" b 🌍"}"#;
+ assert_eq!(
+ JsonMinifier.minify(input).unwrap().as_deref(),
+ Some(compact)
+ );
+ assert_eq!(JsonMinifier.minify(compact).unwrap(), None);
}
#[test]
diff --git a/crates/headroom-core/src/transforms/smart_crusher/crusher.rs b/crates/headroom-core/src/transforms/smart_crusher/crusher.rs
index ff5624760..c626ae240 100644
--- a/crates/headroom-core/src/transforms/smart_crusher/crusher.rs
+++ b/crates/headroom-core/src/transforms/smart_crusher/crusher.rs
@@ -372,14 +372,13 @@ impl SmartCrusher {
///
/// Parses `content` as JSON, recursively processes it (compressing
/// arrays at every depth via the appropriate per-type crusher),
- /// then re-serializes with Python-compatible formatting (`, ` and
- /// `: ` separators, ASCII-escaped non-ASCII).
+ /// then returns original bytes for unchanged values. Transformed
+ /// values use compact JSON, unescaped Unicode, and insertion order.
///
/// Returns a `CrushResult` with:
- /// - `compressed`: the re-serialized JSON.
+ /// - `compressed`: transformed JSON, or original bytes on passthrough.
/// - `original`: the input string (unmodified).
- /// - `was_modified`: whether `compressed` differs from `content`'s
- /// trimmed form.
+ /// - `was_modified`: whether processing changed the parsed JSON value.
/// - `strategy`: combined strategy info from all crushed arrays
/// (or `"passthrough"`).
pub fn crush(&self, content: &str, query: &str, bias: f64) -> CrushResult {
@@ -418,9 +417,9 @@ impl SmartCrusher {
}
}
- /// `SmartCrusher._smart_crush_content` (Python line 2243-2301).
- /// JSON-parse, recursively process, re-serialize. CCR marker
- /// injection is stubbed (CCR is disabled in this stage).
+ /// Apply recursive JSON transforms without rewriting unchanged values.
+ /// Actual transformations use compact JSON; unchanged values retain
+ /// the original bytes, including whitespace and escapes.
///
/// Returns `(crushed_content, was_modified, info)`.
pub fn smart_crush_content(
@@ -446,19 +445,12 @@ impl SmartCrusher {
let (crushed, info) =
self.process_value_with_hook(&parsed, 0, query_context, bias, prose_hook);
- // When an object and its descendants were left unchanged, keep
- // the original bytes. Re-serializing would rewrite Unicode
- // escapes and numeric lexical forms, then advertise a false
- // compression. Nested skip/compaction still serializes below
- // because those paths populate `info`. Spaced JSON without a
- // nested strategy is compact-serialized for historical crush()
- // / parity fixtures.
- if crushed == parsed
- && info.is_empty()
- && matches!(parsed, Value::Object(_))
- && !has_json_insignificant_whitespace(content)
- {
- return (content.to_string(), false, String::new());
+ // An unchanged value is passthrough, regardless of container type
+ // or whitespace. Preserve its bytes, escapes, and numeric forms
+ // instead of advertising re-serialization as compression. Keep
+ // diagnostic strategy information for decisions that made no edit.
+ if crushed == parsed {
+ return (content.to_string(), false, info);
}
// Re-serialize with Python `safe_json_dumps` formatting:
@@ -1295,36 +1287,6 @@ fn hash_canonical(canonical: &str) -> String {
// here because `process_string`'s `string_ccr:<kind>` strategy-info
// label is local to this module's debug-string convention.
-/// True when `s` has JSON whitespace outside of string literals
-/// (spaces after `:` / `,`, pretty-print newlines, etc.). Used to
-/// keep compact unchanged objects on the original-bytes path while
-/// still compact-serializing spaced inputs for historical crush()
-/// output.
-fn has_json_insignificant_whitespace(s: &str) -> bool {
- let mut in_string = false;
- let mut escape = false;
- for ch in s.trim().chars() {
- if in_string {
- if escape {
- escape = false;
- } else if ch == '\\' {
- escape = true;
- } else if ch == '"' {
- in_string = false;
- }
- continue;
- }
- if ch == '"' {
- in_string = true;
- continue;
- }
- if ch.is_whitespace() {
- return true;
- }
- }
- false
-}
-
fn opaque_kind_label(kind: &super::compaction::OpaqueKind) -> &str {
use super::compaction::OpaqueKind;
match kind {
@@ -2004,36 +1966,45 @@ mod tests {
}
#[test]
- fn crush_unchanged_object_keeps_original_bytes() {
+ fn crush_unchanged_values_preserve_original_bytes() {
let c = crusher();
- // Compact input: escaped Unicode and numeric lexical forms must
- // survive the passthrough branch rather than being reserialized
- // (`\u00e9` → `é`, `1.0` → `1`).
- let compact = r#"{"cafe":"caf\u00e9","n":1.0,"ok":true}"#;
- let result = c.crush(compact, "", 1.0);
- assert_eq!(result.compressed, compact);
- assert!(!result.was_modified);
- assert_eq!(result.strategy, "passthrough");
- assert!(
- !result.compressed.contains("<<ccr:"),
- "unchanged object must not invent a CCR marker"
- );
+ let cases = [
+ r#"{"cafe":"caf\u00e9","n":1.0,"ok":true}"#,
+ r#"{ "cafe": "caf\u00e9", "n": 1.0, "ok": true }"#,
+ " \n{\n \"cafe\": \"caf\\u00e9\", \"n\": 1.0\n}\n\t",
+ r#"[ "\u0061", 1.0, { "ok": true } ]"#,
+ " \t\"caf\\u00e9\"\r\n ",
+ " \n1.0e+03\t ",
+ " \tnull\r\n ",
+ ];
+ for input in cases {
+ let result = c.crush(input, "", 1.0);
+ assert_eq!(
+ result.compressed, input,
+ "passthrough must preserve the original representation"
+ );
+ assert!(!result.was_modified);
+ assert_eq!(result.strategy, "passthrough");
+ assert!(
+ !result.compressed.contains("<<ccr:"),
+ "unchanged values must not invent CCR retrieval markers"
+ );
+ }
+ }
- // Spaced / pretty-printed unchanged objects still compact-serialize
- // (historical crush() output, required by parity fixtures) but
- // must not advertise a key-drop strategy or invent a CCR hash.
- let spaced = "{ \"cafe\": \"caf\\u00e9\", \"n\": 1.0, \"ok\": true }";
- let spaced_result = c.crush(spaced, "", 1.0);
- assert_eq!(spaced_result.strategy, "passthrough");
- assert!(
- !spaced_result.strategy.contains("adaptive"),
- "must not report a key-drop strategy: {}",
- spaced_result.strategy
- );
- assert!(!spaced_result.compressed.contains("<<ccr:"));
- let parsed: Value = serde_json::from_str(&spaced_result.compressed).unwrap();
- assert_eq!(parsed["cafe"], "café");
- assert_eq!(parsed["ok"], true);
+ #[test]
+ fn crush_unchanged_analyzed_values_do_not_report_compression() {
+ let c = SmartCrusher::without_compaction(SmartCrusherConfig::default());
+ let rows: Vec<Value> = (0..15)
+ .map(|i| json!({"i": i, "kind": "deep", "v": format!("x{i}")}))
+ .collect();
+ let nested = serde_json::to_string_pretty(&json!({"a": {"b": {"events": rows}}}))
+ .expect("valid nested JSON");
+ for input in ["[null, true, false, null, true, false, null]", &nested] {
+ let result = c.crush(input, "", 1.0);
+ assert_eq!(result.compressed, input);
+ assert!(!result.was_modified);
+ }
}
#[test]
diff --git a/crates/headroom-core/tests/live_zone_ccr.rs b/crates/headroom-core/tests/live_zone_ccr.rs
index 7f973d2d2..c34833b48 100644
--- a/crates/headroom-core/tests/live_zone_ccr.rs
+++ b/crates/headroom-core/tests/live_zone_ccr.rs
@@ -117,3 +117,31 @@ fn store_only_populated_after_token_gate_admits() {
.expect("dispatcher");
assert_eq!(store.len(), 0, "no compression → no CCR put");
}
+
+#[test]
+fn lossless_json_minification_stays_valid_without_a_retrieval_obligation() {
+ let fields: serde_json::Map<String, Value> =
+ (0..500).map(|i| (format!("field_{i}"), json!(i))).collect();
+ let value = json!([fields]);
+ let payload = serde_json::to_string_pretty(&value).unwrap();
+ let body = body_with_payload(&payload);
+ let store = InMemoryCcrStore::new();
+ let outcome = compress_anthropic_live_zone_with_ccr(
+ &body,
+ 0,
+ AuthMode::Payg,
+ DEFAULT_MODEL,
+ Some(&store),
+ )
+ .unwrap();
+ let LiveZoneOutcome::Modified { new_body, .. } = outcome else {
+ panic!("pretty JSON must save tokens through the lossless minifier");
+ };
+ let forwarded: Value = serde_json::from_str(new_body.get()).unwrap();
+ let content = forwarded["messages"][0]["content"][0]["content"]
+ .as_str()
+ .unwrap();
+ assert_eq!(serde_json::from_str::<Value>(content).unwrap(), value);
+ assert_eq!(content, serde_json::to_string(&value).unwrap());
+ assert!(store.get(&compute_key(payload.as_bytes())).is_none());
+}
diff --git a/crates/headroom-py/src/lib.rs b/crates/headroom-py/src/lib.rs
index 52d9519a5..c56739d7e 100644
--- a/crates/headroom-py/src/lib.rs
+++ b/crates/headroom-py/src/lib.rs
@@ -18,6 +18,7 @@ use std::collections::BTreeMap;
use headroom_core::signals::{
ImportanceCategory, ImportanceContext, KeywordDetector, KeywordRegistry, LineImportanceDetector,
};
+use headroom_core::transforms::pipeline::JsonMinifier;
use headroom_core::transforms::smart_crusher::compaction::{
ClassifyConfig, CompactConfig, DocumentCompactor,
};
@@ -47,6 +48,7 @@ use headroom_core::transforms::{
TextCrusherResult as RustTextCrusherResult,
};
use pyo3::prelude::*;
+use pyo3::pybacked::PyBackedStr;
use pyo3::types::{PyBytes, PyDict};
/// Identity stub used by the Python smoke test to verify linkage.
@@ -55,6 +57,13 @@ fn hello() -> &'static str {
headroom_core::hello()
}
+/// Run the explicit JSON reformat without copying the immutable Python input.
+#[pyfunction]
+fn minify_json(py: Python<'_>, content: PyBackedStr) -> PyResult<Option<String>> {
+ py.detach(|| JsonMinifier.minify(content.as_str()))
+ .map_err(|error| pyo3::exceptions::PyValueError::new_err(error.to_string()))
+}
+
fn type_name(v: &serde_json::Value) -> &'static str {
match v {
serde_json::Value::Null => "null",
@@ -1814,6 +1823,7 @@ fn _core(m: &Bound<'_, PyModule>) -> PyResult<()> {
let _ = pyo3_log::try_init();
m.add_function(wrap_pyfunction!(hello, m)?)?;
+ m.add_function(wrap_pyfunction!(minify_json, m)?)?;
m.add_class::<PyDiffCompressorConfig>()?;
m.add_class::<PyDiffCompressionResult>()?;
m.add_class::<PyDiffCompressorStats>()?;
diff --git a/docs/content/docs/smart-crusher.mdx b/docs/content/docs/smart-crusher.mdx
index cd7e335f4..291866969 100644
--- a/docs/content/docs/smart-crusher.mdx
+++ b/docs/content/docs/smart-crusher.mdx
@@ -32,6 +32,61 @@ On that path, the target size is `max_items_after_crush` (15 by default): a 1,00
| Relevant | Top K | Match user's query |
| Others | Sampled | Statistical representation |
+### JSON objects and unchanged values
+
+Object keys are never sampled or dropped: named fields are not interchangeable
+array rows. Removing a key without a retrieval marker would make its value
+unrecoverable. SmartCrusher instead recurses into each value, so nested arrays
+can still use lossless table compaction or CCR-backed row offload, and opaque
+strings can still be replaced by retrievable CCR references.
+
+A wide object containing only unchanged values therefore has **zero structural
+compression savings**, regardless of how many keys it contains. This is a
+deliberate reversibility tradeoff, not a promise that every large JSON object
+will shrink. Key-level offload would need its own retrieval contract before
+object-key sampling could safely return.
+
+When recursive processing leaves a valid JSON value unchanged, SmartCrusher
+returns its original bytes and sets `was_modified=False`. This applies to
+objects, arrays, and scalars, including whitespace, Unicode escapes, and numeric
+representations. Actual structural transformations still use compact JSON
+serialization.
+
+The routing pipeline then applies the existing Rust `JsonMinifier` as a separate
+lossless step when SmartCrusher leaves routed JSON unchanged. Savings are
+attributed to `json_minifier`, not `smart_crusher`, and do not create structural
+TOIN records. Minification removes only whitespace outside strings: key order,
+duplicate keys, string escapes, and number spellings remain byte-stable. Repeating
+the same input produces the same output; already-minified JSON is a no-op.
+The native live-zone JSON-array arm follows the same explicit step; its existing
+content-type and token-reduction gates still apply.
+Because no information is offloaded, lossless minification does not store an
+original or append a CCR retrieval marker: the forwarded value remains valid
+JSON. A mixed document still reports `mixed` when its sections take different
+routes, even if only its JSON section saves tokens.
+Minification takes precedence over lossy fallbacks when it saves tokens;
+already-compact input can still reach the existing fallback chain. For tool
+results with CCR enabled, each changed lossy mixed section must carry its own
+retrieval marker. Embedded JSON carries reversibility separately for each adopted
+span: lossless minification needs no marker, while a marker on another span or
+section does not authorize dropping its data. Candidates rejected by the token
+benefit gate do not affect the adopted spans' reversibility. Rejected mixed
+results do not enter the tool-result cache.
+
+### Comparing historical TOIN data
+
+Unchanged values do not produce new TOIN compression records, even when the
+strategy contains a diagnostic about a skipped transformation. Older TOIN
+aggregates may include object-scoring strategies, formatting-only changes, or
+zero-savings `lossless_only` decisions from different implementations.
+
+Do not compare those aggregates directly with current object-key-preserving
+results or treat zero-savings decisions as successful compression. Compare the
+same corpus and configuration on both revisions, report unchanged wide objects
+separately from objects with transformed descendants, and identify the engine
+revision. Existing TOIN history is retained; this change does not rewrite or
+delete a user's learned state.
+
## Quick Start
<Tabs groupId="lang" items={['TypeScript', 'Python']}>
diff --git a/headroom/transforms/content_router.py b/headroom/transforms/content_router.py
index 994dd341d..3359f0c75 100644
--- a/headroom/transforms/content_router.py
+++ b/headroom/transforms/content_router.py
@@ -2031,6 +2031,7 @@ class CompressionStrategy(Enum):
CODE_AWARE = "code_aware"
SMART_CRUSHER = "smart_crusher"
+ JSON_MINIFIER = "json_minifier"
SEARCH = "search"
LOG = "log"
KOMPRESS = "kompress"
@@ -2117,6 +2118,8 @@ class RouterCompressionResult:
LOG fallback chain it's three. Lets log readers see *how*
we got to the final compressor without parsing the
decision_reason string.
+ unrecoverable_sections: A changed lossy mixed section or adopted embedded
+ JSON span lacks its own recovery, independently of sibling markers.
cache_hit: True when this result was reused from a cache
instead of a fresh compression run. compress() itself
never sets this (only apply() has the router-internal
@@ -2132,6 +2135,7 @@ class RouterCompressionResult:
sections_processed: int = 1
strategy_chain: list[str] = field(default_factory=list)
cache_hit: bool = False
+ unrecoverable_sections: bool = False
@property
def total_original_tokens(self) -> int:
@@ -3171,8 +3175,8 @@ class ContentRouter(Transform):
language: Optional language hint for code.
context: Query context for pattern learning.
"""
- # Skip SmartCrusher - it handles its own TOIN recording
- if strategy == CompressionStrategy.SMART_CRUSHER:
+ # SmartCrusher records its own decisions; reformatting does not train TOIN.
+ if strategy in {CompressionStrategy.SMART_CRUSHER, CompressionStrategy.JSON_MINIFIER}:
return
# Skip if no actual compression happened
@@ -3599,6 +3603,7 @@ class ContentRouter(Transform):
compressed_sections: list[str] = []
routing_log: list[RoutingDecision] = []
+ unrecoverable_sections = False
for i, section in enumerate(sections):
if placeholders and any(ph in section.content for ph in placeholders):
@@ -3626,15 +3631,27 @@ class ContentRouter(Transform):
# Compress section
original_tokens = _estimate_tokens(section.content)
- compressed_content, compressed_tokens, _section_chain = self._apply_strategy_to_content(
- section.content,
- strategy,
- context,
- section.language,
- question,
- bias=bias,
+ compressed_content, compressed_tokens, _section_chain, reversible = (
+ self._apply_strategy_to_content(
+ section.content,
+ strategy,
+ context,
+ section.language,
+ question,
+ bias=bias,
+ )
)
+ if _section_chain == [CompressionStrategy.JSON_MINIFIER.value]:
+ strategy = CompressionStrategy.JSON_MINIFIER
+ if (
+ compressed_content != section.content
+ and not reversible
+ and (
+ strategy in self.LOSSY_UNMARKED_STRATEGIES or "embedded_json" in _section_chain
+ )
+ ):
+ unrecoverable_sections = True
# Preserve code fence markers
if section.is_code_fence and section.language:
compressed_content = f"```{section.language}\n{compressed_content}\n```"
@@ -3671,13 +3688,19 @@ class ContentRouter(Transform):
compressed = "\n\n".join(compressed_sections)
if protected:
compressed = restore_tags(compressed, protected)
+ routed_strategies = {decision.strategy for decision in routing_log}
return RouterCompressionResult(
compressed=compressed,
original=content,
- strategy_used=CompressionStrategy.MIXED,
+ strategy_used=(
+ next(iter(routed_strategies))
+ if len(routed_strategies) == 1
+ else CompressionStrategy.MIXED
+ ),
routing_log=routing_log,
sections_processed=len(sections),
+ unrecoverable_sections=unrecoverable_sections,
)
def _compress_pure(
@@ -3702,15 +3725,18 @@ class ContentRouter(Transform):
"""
original_tokens = _estimate_tokens(content)
- compressed, compressed_tokens, strategy_chain = self._apply_strategy_to_content(
+ compressed, compressed_tokens, strategy_chain, reversible = self._apply_strategy_to_content(
content, strategy, context, question=question, bias=bias
)
+ if strategy_chain == [CompressionStrategy.JSON_MINIFIER.value]:
+ strategy = CompressionStrategy.JSON_MINIFIER
return RouterCompressionResult(
compressed=compressed,
original=content,
strategy_used=strategy,
strategy_chain=strategy_chain,
+ unrecoverable_sections=not reversible and "embedded_json" in strategy_chain,
routing_log=[
RoutingDecision(
content_type=self._content_type_from_strategy(strategy),
@@ -4252,7 +4278,7 @@ class ContentRouter(Transform):
question: str | None = None,
bias: float = 1.0,
_allow_embedded: bool = True,
- ) -> tuple[str, int, list[str]]:
+ ) -> tuple[str, int, list[str], bool]:
"""Apply a compression strategy to content.
Args:
@@ -4265,8 +4291,11 @@ class ContentRouter(Transform):
Returns:
Tuple of (compressed_content, compressed_token_count,
- strategy_chain). The chain lists every strategy attempted
- in order — first the requested one, then any fallbacks.
+ strategy_chain, reversible). Reversible means unchanged, known
+ lossless, or covered by this result's own retrieval marker(s).
+ Embedded routing aggregates only adopted spans; a marker in one
+ span cannot cover another span's unmarked loss.
+ The chain lists every strategy attempted in order.
Single-entry chain means a direct hit; multi-entry means
the fallback chain fired (e.g. ``[smart_crusher, kompress,
log]``). Log readers use this to see *how* we got to the
@@ -4275,7 +4304,7 @@ class ContentRouter(Transform):
# Git status is path/state ground truth, not a rectangular CSV table.
# Preserve it before embedded-JSON, lossy-after-fold and external stages.
if _try_detect_git_status(_strip_detection_envelope(content)) is not None:
- return content, _estimate_tokens(content), [CompressionStrategy.TABULAR.value]
+ return content, _estimate_tokens(content), [CompressionStrategy.TABULAR.value], True
# ── STRUCTURAL (embedded) JSON routing ───────────────────────────────
# Before anything else: if this block is not a single JSON value but
@@ -4304,7 +4333,8 @@ class ContentRouter(Transform):
if _allow_embedded and not _defer_embedded_for_html:
routed = self._embedded_json_route(content, context, question, bias)
if routed is not None:
- return routed, _estimate_tokens(routed), ["embedded_json"]
+ text, reversible = routed
+ return text, _estimate_tokens(text), ["embedded_json"], reversible
# Track original tokens for TOIN recording
original_tokens = _estimate_tokens(content)
@@ -4336,8 +4366,8 @@ class ContentRouter(Transform):
# never a marker-free lossy drop that could not be recovered.
if self.config.lossless:
if _ll_label is not None:
- return _ll_content, _estimate_tokens(_ll_content), [_ll_label]
- return content, original_tokens, [CompressionStrategy.PASSTHROUGH.value]
+ return _ll_content, _estimate_tokens(_ll_content), [_ll_label], True
+ return content, original_tokens, [CompressionStrategy.PASSTHROUGH.value], True
# ── LOSSY / CCR mode: layer relevance-split + lossy ON TOP of the fold ─
# The operator has opted into lossy compression, so we reclaim more than
@@ -4360,7 +4390,12 @@ class ContentRouter(Transform):
kind = "log" if strategy is CompressionStrategy.LOG else "search"
split = self._relevance_split_compress(content, kind, context)
if split is not None:
- return split, _estimate_tokens(split), [kind, "relevance_split"]
+ return (
+ split,
+ _estimate_tokens(split),
+ [kind, "relevance_split"],
+ bool(CCR_RETRIEVAL_MARKER_RE.search(split)),
+ )
# No relevance split adopted → return the STAGE 0 lossless fold as the
# floor. Lossless-then-lossy: before returning, run the aggressive lossy
@@ -4393,8 +4428,9 @@ class ContentRouter(Transform):
_komp,
_komp_tokens,
[_ll_label, CompressionStrategy.KOMPRESS.value],
+ bool(CCR_RETRIEVAL_MARKER_RE.search(_komp)),
)
- return _ll_content, _estimate_tokens(_ll_content), [_ll_label]
+ return _ll_content, _estimate_tokens(_ll_content), [_ll_label], True
# CCR/lossy mode, nothing foldable (code/json/text/mixed) and no relevance
# split → fall through to the lossy compressors below (kompress /
@@ -4411,7 +4447,7 @@ class ContentRouter(Transform):
# through to the EXISTING built-in dispatch UNCHANGED.
external = self._try_external_compressor(content, strategy, context, question)
if external is not None:
- return external
+ return (*external, bool(CCR_RETRIEVAL_MARKER_RE.search(external[0])))
try:
if strategy == CompressionStrategy.CODE_AWARE:
@@ -4503,6 +4539,18 @@ class ContentRouter(Transform):
compressed = output.content
compressed_tokens = _estimate_tokens(output.content)
decision_reason = "smart_crusher"
+ if compressed == content:
+ from headroom._core import minify_json
+
+ minified = minify_json(content)
+ if minified is not None:
+ compressed = minified
+ compressed_tokens = _estimate_tokens(minified)
+ strategy = CompressionStrategy.JSON_MINIFIER
+ actual_strategy = strategy
+ strategy_chain = [strategy.value]
+ compressor_name = "JsonMinifier"
+ decision_reason = "json_minifier"
# Fallback to Kompress (and possibly Log) is
# handled by the unified post-strategy block below
# — no inline fallback here to avoid duplicate
@@ -4850,7 +4898,12 @@ class ContentRouter(Transform):
language=language,
context=context,
)
- return compressed, compressed_tokens, strategy_chain
+ reversible = (
+ compressed == content
+ or strategy_chain == [CompressionStrategy.JSON_MINIFIER.value]
+ or bool(CCR_RETRIEVAL_MARKER_RE.search(compressed))
+ )
+ return compressed, compressed_tokens, strategy_chain, reversible
# #3609: deferred embedded-JSON splice. The HTML strategy ran above and
# extracted nothing (``compressed`` is still None), so the local JSON
@@ -4860,7 +4913,8 @@ class ContentRouter(Transform):
routed = self._embedded_json_route(content, context, question, bias)
if routed is not None:
strategy_chain.append("embedded_json")
- return routed, _estimate_tokens(routed), strategy_chain
+ text, reversible = routed
+ return text, _estimate_tokens(text), strategy_chain, reversible
# Fallback: return unchanged
if self.config.enable_dense_line_elision:
@@ -4877,7 +4931,12 @@ class ContentRouter(Transform):
language=language,
context=context,
)
- return elided, elided_tokens, strategy_chain
+ return (
+ elided,
+ elided_tokens,
+ strategy_chain,
+ bool(CCR_RETRIEVAL_MARKER_RE.search(elided)),
+ )
strategy_chain.append(CompressionStrategy.PASSTHROUGH.value)
if _router_debug_enabled():
_log_router_debug(
@@ -4899,7 +4958,7 @@ class ContentRouter(Transform):
output=content,
error=error,
)
- return content, original_tokens, strategy_chain
+ return content, original_tokens, strategy_chain, True
def _elide_dense(self, text: str, context: str) -> tuple[str, int] | None:
"""Dense-line elision with a CCR retrieval marker; None when no line is dense.
@@ -4959,19 +5018,19 @@ class ContentRouter(Transform):
def _embedded_json_route(
self, content: str, context: str, question: str | None, bias: float
- ) -> str | None:
+ ) -> tuple[str, bool] | None:
"""Route embedded JSON spans inside ``content`` through the JSON compressors.
Extracted from the ``_apply_strategy_to_content`` pre-pass so the HTML
strategy can defer the splice (#3609) and retry it in the passthrough
- fallback. Returns the spliced block, or ``None`` when nothing
- safe/smaller applied.
+ fallback. Returns the spliced block and its per-span reversibility,
+ or None when nothing smaller applied.
"""
from headroom.transforms.recursive_json import route_embedded_json
- def _dispatch_span(span: str) -> str | None:
+ def _dispatch_span(span: str) -> tuple[str, bool] | None:
strat = self._strategy_from_detection_type(_detect_content(span).content_type)
- text, _t, _c = self._apply_strategy_to_content(
+ text, _t, _c, reversible = self._apply_strategy_to_content(
span,
strat,
context,
@@ -4979,7 +5038,7 @@ class ContentRouter(Transform):
bias=bias,
_allow_embedded=False,
)
- return text if text != span else None
+ return (text, reversible) if text != span else None
return route_embedded_json(content, _dispatch_span, tok=_estimate_tokens)
@@ -5286,6 +5345,7 @@ class ContentRouter(Transform):
mapping = {
CompressionStrategy.CODE_AWARE: ContentType.SOURCE_CODE,
CompressionStrategy.SMART_CRUSHER: ContentType.JSON_ARRAY,
+ CompressionStrategy.JSON_MINIFIER: ContentType.JSON_ARRAY,
CompressionStrategy.SEARCH: ContentType.SEARCH_RESULTS,
CompressionStrategy.LOG: ContentType.BUILD_OUTPUT,
CompressionStrategy.DIFF: ContentType.GIT_DIFF,
@@ -7170,8 +7230,13 @@ class ContentRouter(Transform):
if (
enforce_rev
and self.config.ccr_inject_marker
- and result.strategy_used in self.LOSSY_UNMARKED_STRATEGIES
- and not CCR_RETRIEVAL_MARKER_RE.search(result.compressed)
+ and (
+ getattr(result, "unrecoverable_sections", False)
+ or (
+ result.strategy_used in self.LOSSY_UNMARKED_STRATEGIES
+ and not CCR_RETRIEVAL_MARKER_RE.search(result.compressed)
+ )
+ )
):
self._cache.mark_skip(content_key)
result_slots[slot_idx] = message
@@ -8313,8 +8378,13 @@ class ContentRouter(Transform):
if (
enforce_reversibility
and self.config.ccr_inject_marker
- and result.strategy_used in self.LOSSY_UNMARKED_STRATEGIES
- and not CCR_RETRIEVAL_MARKER_RE.search(result.compressed)
+ and (
+ getattr(result, "unrecoverable_sections", False)
+ or (
+ result.strategy_used in self.LOSSY_UNMARKED_STRATEGIES
+ and not CCR_RETRIEVAL_MARKER_RE.search(result.compressed)
+ )
+ )
):
self._cache.mark_skip(content_key)
if route_counts is not None:
diff --git a/headroom/transforms/recursive_json.py b/headroom/transforms/recursive_json.py
index 539536b85..76e6ef1c1 100644
--- a/headroom/transforms/recursive_json.py
+++ b/headroom/transforms/recursive_json.py
@@ -37,9 +37,9 @@ _OPEN = "[{"
_CLOSE = "]}"
_PAIR = {"}": "{", "]": "["}
-#: A dispatch callback: given a JSON span's text, return the compressed text
-#: (which may carry CCR markers) or ``None`` to leave it unchanged.
-Dispatch = Callable[[str], "str | None"]
+#: Return (compressed text, reversible) or None to leave the span unchanged.
+#: Reversibility must cover this span, not a marker elsewhere in the document.
+Dispatch = Callable[[str], "tuple[str, bool] | None"]
def _match_span(text: str, start: int) -> int | None:
@@ -240,10 +240,10 @@ def route_embedded_json(
dispatch: Dispatch,
*,
tok: Callable[[str], int] | None = None,
-) -> str | None:
- """Route every embedded JSON span in ``content`` through ``dispatch`` and
- splice the results back in place. Returns the rewritten block, or ``None``
- when nothing safe/smaller applied.
+) -> tuple[str, bool] | None:
+ """Splice smaller dispatch results into ``content``, preserving surrounding bytes.
+
+ Return (rewritten block, all adopted spans reversible), or None if no win.
``content`` that is itself a single JSON value is intentionally skipped — the
caller already routes pure-JSON blocks; this exists for the *embedded* case.
@@ -257,17 +257,22 @@ def route_embedded_json(
return None
repls: list[tuple[int, int, str]] = []
+ reversible = True
for a, b in spans:
chunk = content[a:b]
if "<<ccr:" in chunk: # R1: already compressed — never re-route
continue
if not carries_record_array(chunk):
continue
- out = dispatch(chunk)
- if out is None or out == chunk:
+ result = dispatch(chunk)
+ if result is None:
+ continue
+ out, span_reversible = result
+ if out == chunk:
continue
if tok(out) < tok(chunk): # benefit gate (outcome, not a threshold)
repls.append((a, b, out))
+ reversible = reversible and span_reversible
if not repls:
return None
@@ -279,4 +284,4 @@ def route_embedded_json(
last = b
parts.append(content[last:])
new = "".join(parts)
- return new if tok(new) < tok(content) else None
+ return (new, reversible) if tok(new) < tok(content) else None
diff --git a/headroom/transforms/smart_crusher.py b/headroom/transforms/smart_crusher.py
index 2cd3db299..f1991ddb1 100644
--- a/headroom/transforms/smart_crusher.py
+++ b/headroom/transforms/smart_crusher.py
@@ -550,11 +550,9 @@ class SmartCrusher(Transform):
# Bridging the gap here keeps JSON-array compressions fueling
# the learning system.
#
- # Filter on `was_modified AND strategy != "passthrough"`. The
- # Rust crusher sometimes flips `was_modified=True` from pure
- # JSON re-canonicalization (whitespace normalization) without
- # actually compressing — the strategy stays `"passthrough"` in
- # that case, and there's no learning value in recording it.
+ # Record actual transformations with a non-passthrough strategy.
+ # Rust preserves original bytes when the parsed value is unchanged,
+ # including no-op decisions that retain diagnostic strategy info.
if record_toin and r.was_modified and r.strategy != "passthrough":
self._record_to_toin(
original=content,
@@ -906,9 +904,8 @@ class SmartCrusher(Transform):
tool_name=tool_name,
):
return content, False, "passthrough"
- # Same passthrough filter as `crush()` — re-canonicalization of
- # JSON whitespace can flip `was_modified=True` even when the
- # `info` field reports `passthrough` and no compression happened.
+ # Match `crush()`: unchanged values must not train TOIN, even when
+ # `info` contains a diagnostic about a skipped transformation.
if was_modified and info != "passthrough":
self._record_to_toin(
original=content,
diff --git a/tests/parity/fixtures/smart_crusher/nested_3deep_with_array_bb5036c153aa.json b/tests/parity/fixtures/smart_crusher/nested_3deep_with_array_9aeaf457a2ba.json
similarity index 64%
rename from tests/parity/fixtures/smart_crusher/nested_3deep_with_array_bb5036c153aa.json
rename to tests/parity/fixtures/smart_crusher/nested_3deep_with_array_9aeaf457a2ba.json
index 3c0638562..b8a20a9cf 100644
--- a/tests/parity/fixtures/smart_crusher/nested_3deep_with_array_bb5036c153aa.json
+++ b/tests/parity/fixtures/smart_crusher/nested_3deep_with_array_9aeaf457a2ba.json
@@ -1,11 +1,18 @@
{
"config": {
+ "compaction_core_field_fraction": 0.8,
+ "compaction_heterogeneous_core_ratio": 0.6,
+ "compaction_max_buckets": 8,
+ "compaction_max_flatten_inner_keys": 6,
+ "compaction_min_buckets": 2,
"dedup_identical_items": true,
"enabled": true,
"factor_out_constants": false,
"first_fraction": 0.3,
"include_summaries": false,
"last_fraction": 0.15,
+ "lossless_min_savings_ratio": 0.15,
+ "lossless_only": false,
"max_items_after_crush": 15,
"min_items_to_analyze": 5,
"min_tokens_to_crush": 200,
@@ -21,14 +28,14 @@
"content": "{\"a\": {\"b\": {\"events\": [{\"i\": 0, \"kind\": \"deep\", \"v\": \"x0\"}, {\"i\": 1, \"kind\": \"deep\", \"v\": \"x1\"}, {\"i\": 2, \"kind\": \"deep\", \"v\": \"x2\"}, {\"i\": 3, \"kind\": \"deep\", \"v\": \"x3\"}, {\"i\": 4, \"kind\": \"deep\", \"v\": \"x4\"}, {\"i\": 5, \"kind\": \"deep\", \"v\": \"x5\"}, {\"i\": 6, \"kind\": \"deep\", \"v\": \"x6\"}, {\"i\": 7, \"kind\": \"deep\", \"v\": \"x7\"}, {\"i\": 8, \"kind\": \"deep\", \"v\": \"x8\"}, {\"i\": 9, \"kind\": \"deep\", \"v\": \"x9\"}, {\"i\": 10, \"kind\": \"deep\", \"v\": \"x10\"}, {\"i\": 11, \"kind\": \"deep\", \"v\": \"x11\"}, {\"i\": 12, \"kind\": \"deep\", \"v\": \"x12\"}, {\"i\": 13, \"kind\": \"deep\", \"v\": \"x13\"}, {\"i\": 14, \"kind\": \"deep\", \"v\": \"x14\"}]}}}",
"query": ""
},
- "input_sha256": "bb5036c153aa8a9e6700b751413952c4753c328cc8bd63986f68c3a5b4fb9cb6",
+ "input_sha256": "9aeaf457a2bad5b296e27c04ae7e2af9501c046063f0f3a145dec295605c41a1",
"label": "nested_3deep_with_array",
"output": {
- "compressed": "{\"a\":{\"b\":{\"events\":[{\"i\":0,\"kind\":\"deep\",\"v\":\"x0\"},{\"i\":1,\"kind\":\"deep\",\"v\":\"x1\"},{\"i\":2,\"kind\":\"deep\",\"v\":\"x2\"},{\"i\":3,\"kind\":\"deep\",\"v\":\"x3\"},{\"i\":4,\"kind\":\"deep\",\"v\":\"x4\"},{\"i\":5,\"kind\":\"deep\",\"v\":\"x5\"},{\"i\":6,\"kind\":\"deep\",\"v\":\"x6\"},{\"i\":7,\"kind\":\"deep\",\"v\":\"x7\"},{\"i\":8,\"kind\":\"deep\",\"v\":\"x8\"},{\"i\":9,\"kind\":\"deep\",\"v\":\"x9\"},{\"i\":10,\"kind\":\"deep\",\"v\":\"x10\"},{\"i\":11,\"kind\":\"deep\",\"v\":\"x11\"},{\"i\":12,\"kind\":\"deep\",\"v\":\"x12\"},{\"i\":13,\"kind\":\"deep\",\"v\":\"x13\"},{\"i\":14,\"kind\":\"deep\",\"v\":\"x14\"}]}}}",
+ "compressed": "{\"a\": {\"b\": {\"events\": [{\"i\": 0, \"kind\": \"deep\", \"v\": \"x0\"}, {\"i\": 1, \"kind\": \"deep\", \"v\": \"x1\"}, {\"i\": 2, \"kind\": \"deep\", \"v\": \"x2\"}, {\"i\": 3, \"kind\": \"deep\", \"v\": \"x3\"}, {\"i\": 4, \"kind\": \"deep\", \"v\": \"x4\"}, {\"i\": 5, \"kind\": \"deep\", \"v\": \"x5\"}, {\"i\": 6, \"kind\": \"deep\", \"v\": \"x6\"}, {\"i\": 7, \"kind\": \"deep\", \"v\": \"x7\"}, {\"i\": 8, \"kind\": \"deep\", \"v\": \"x8\"}, {\"i\": 9, \"kind\": \"deep\", \"v\": \"x9\"}, {\"i\": 10, \"kind\": \"deep\", \"v\": \"x10\"}, {\"i\": 11, \"kind\": \"deep\", \"v\": \"x11\"}, {\"i\": 12, \"kind\": \"deep\", \"v\": \"x12\"}, {\"i\": 13, \"kind\": \"deep\", \"v\": \"x13\"}, {\"i\": 14, \"kind\": \"deep\", \"v\": \"x14\"}]}}}",
"original": "{\"a\": {\"b\": {\"events\": [{\"i\": 0, \"kind\": \"deep\", \"v\": \"x0\"}, {\"i\": 1, \"kind\": \"deep\", \"v\": \"x1\"}, {\"i\": 2, \"kind\": \"deep\", \"v\": \"x2\"}, {\"i\": 3, \"kind\": \"deep\", \"v\": \"x3\"}, {\"i\": 4, \"kind\": \"deep\", \"v\": \"x4\"}, {\"i\": 5, \"kind\": \"deep\", \"v\": \"x5\"}, {\"i\": 6, \"kind\": \"deep\", \"v\": \"x6\"}, {\"i\": 7, \"kind\": \"deep\", \"v\": \"x7\"}, {\"i\": 8, \"kind\": \"deep\", \"v\": \"x8\"}, {\"i\": 9, \"kind\": \"deep\", \"v\": \"x9\"}, {\"i\": 10, \"kind\": \"deep\", \"v\": \"x10\"}, {\"i\": 11, \"kind\": \"deep\", \"v\": \"x11\"}, {\"i\": 12, \"kind\": \"deep\", \"v\": \"x12\"}, {\"i\": 13, \"kind\": \"deep\", \"v\": \"x13\"}, {\"i\": 14, \"kind\": \"deep\", \"v\": \"x14\"}]}}}",
"strategy": "none:adaptive_at_limit(15->15)",
- "was_modified": true
+ "was_modified": false
},
- "recorded_at": "2026-04-27T06:59:41.857099+00:00",
+ "recorded_at": "2026-10-08T19:16:38.005664+00:00",
"transform": "smart_crusher"
}
diff --git a/tests/parity/fixtures/smart_crusher/nested_object_with_array_b66d43299ea1.json b/tests/parity/fixtures/smart_crusher/nested_object_with_array_e257ab3bfe9a.json
similarity index 65%
rename from tests/parity/fixtures/smart_crusher/nested_object_with_array_b66d43299ea1.json
rename to tests/parity/fixtures/smart_crusher/nested_object_with_array_e257ab3bfe9a.json
index 3c5c22c7d..afb2f9353 100644
--- a/tests/parity/fixtures/smart_crusher/nested_object_with_array_b66d43299ea1.json
+++ b/tests/parity/fixtures/smart_crusher/nested_object_with_array_e257ab3bfe9a.json
@@ -1,11 +1,18 @@
{
"config": {
+ "compaction_core_field_fraction": 0.8,
+ "compaction_heterogeneous_core_ratio": 0.6,
+ "compaction_max_buckets": 8,
+ "compaction_max_flatten_inner_keys": 6,
+ "compaction_min_buckets": 2,
"dedup_identical_items": true,
"enabled": true,
"factor_out_constants": false,
"first_fraction": 0.3,
"include_summaries": false,
"last_fraction": 0.15,
+ "lossless_min_savings_ratio": 0.15,
+ "lossless_only": false,
"max_items_after_crush": 15,
"min_items_to_analyze": 5,
"min_tokens_to_crush": 200,
@@ -21,14 +28,14 @@
"content": "{\"request_id\": \"req-1\", \"events\": [{\"step\": 0, \"kind\": \"trace\", \"msg\": \"e0\"}, {\"step\": 1, \"kind\": \"trace\", \"msg\": \"e1\"}, {\"step\": 2, \"kind\": \"trace\", \"msg\": \"e2\"}, {\"step\": 3, \"kind\": \"trace\", \"msg\": \"e3\"}, {\"step\": 4, \"kind\": \"trace\", \"msg\": \"e4\"}, {\"step\": 5, \"kind\": \"trace\", \"msg\": \"e5\"}, {\"step\": 6, \"kind\": \"trace\", \"msg\": \"e6\"}, {\"step\": 7, \"kind\": \"trace\", \"msg\": \"e7\"}, {\"step\": 8, \"kind\": \"trace\", \"msg\": \"e8\"}, {\"step\": 9, \"kind\": \"trace\", \"msg\": \"e9\"}, {\"step\": 10, \"kind\": \"trace\", \"msg\": \"e10\"}, {\"step\": 11, \"kind\": \"trace\", \"msg\": \"e11\"}, {\"step\": 12, \"kind\": \"trace\", \"msg\": \"e12\"}, {\"step\": 13, \"kind\": \"trace\", \"msg\": \"e13\"}, {\"step\": 14, \"kind\": \"trace\", \"msg\": \"e14\"}, {\"step\": 15, \"kind\": \"trace\", \"msg\": \"e15\"}, {\"step\": 16, \"kind\": \"trace\", \"msg\": \"e16\"}, {\"step\": 17, \"kind\": \"trace\", \"msg\": \"e17\"}, {\"step\": 18, \"kind\": \"trace\", \"msg\": \"e18\"}, {\"step\": 19, \"kind\": \"trace\", \"msg\": \"e19\"}]}",
"query": ""
},
- "input_sha256": "b66d43299ea15229c3fea6607b8fc697e32e72811ce4364c71bc9f64357208f1",
+ "input_sha256": "e257ab3bfe9a8a599e4516598d73de4c687f2afaf99e0fa5deca3eb3808d7841",
"label": "nested_object_with_array",
"output": {
- "compressed": "{\"request_id\":\"req-1\",\"events\":[{\"step\":0,\"kind\":\"trace\",\"msg\":\"e0\"},{\"step\":1,\"kind\":\"trace\",\"msg\":\"e1\"},{\"step\":2,\"kind\":\"trace\",\"msg\":\"e2\"},{\"step\":3,\"kind\":\"trace\",\"msg\":\"e3\"},{\"step\":4,\"kind\":\"trace\",\"msg\":\"e4\"},{\"step\":5,\"kind\":\"trace\",\"msg\":\"e5\"},{\"step\":6,\"kind\":\"trace\",\"msg\":\"e6\"},{\"step\":7,\"kind\":\"trace\",\"msg\":\"e7\"},{\"step\":8,\"kind\":\"trace\",\"msg\":\"e8\"},{\"step\":9,\"kind\":\"trace\",\"msg\":\"e9\"},{\"step\":10,\"kind\":\"trace\",\"msg\":\"e10\"},{\"step\":11,\"kind\":\"trace\",\"msg\":\"e11\"},{\"step\":12,\"kind\":\"trace\",\"msg\":\"e12\"},{\"step\":13,\"kind\":\"trace\",\"msg\":\"e13\"},{\"step\":14,\"kind\":\"trace\",\"msg\":\"e14\"},{\"step\":15,\"kind\":\"trace\",\"msg\":\"e15\"},{\"step\":16,\"kind\":\"trace\",\"msg\":\"e16\"},{\"step\":17,\"kind\":\"trace\",\"msg\":\"e17\"},{\"step\":18,\"kind\":\"trace\",\"msg\":\"e18\"},{\"step\":19,\"kind\":\"trace\",\"msg\":\"e19\"}]}",
+ "compressed": "{\"request_id\": \"req-1\", \"events\": [{\"step\": 0, \"kind\": \"trace\", \"msg\": \"e0\"}, {\"step\": 1, \"kind\": \"trace\", \"msg\": \"e1\"}, {\"step\": 2, \"kind\": \"trace\", \"msg\": \"e2\"}, {\"step\": 3, \"kind\": \"trace\", \"msg\": \"e3\"}, {\"step\": 4, \"kind\": \"trace\", \"msg\": \"e4\"}, {\"step\": 5, \"kind\": \"trace\", \"msg\": \"e5\"}, {\"step\": 6, \"kind\": \"trace\", \"msg\": \"e6\"}, {\"step\": 7, \"kind\": \"trace\", \"msg\": \"e7\"}, {\"step\": 8, \"kind\": \"trace\", \"msg\": \"e8\"}, {\"step\": 9, \"kind\": \"trace\", \"msg\": \"e9\"}, {\"step\": 10, \"kind\": \"trace\", \"msg\": \"e10\"}, {\"step\": 11, \"kind\": \"trace\", \"msg\": \"e11\"}, {\"step\": 12, \"kind\": \"trace\", \"msg\": \"e12\"}, {\"step\": 13, \"kind\": \"trace\", \"msg\": \"e13\"}, {\"step\": 14, \"kind\": \"trace\", \"msg\": \"e14\"}, {\"step\": 15, \"kind\": \"trace\", \"msg\": \"e15\"}, {\"step\": 16, \"kind\": \"trace\", \"msg\": \"e16\"}, {\"step\": 17, \"kind\": \"trace\", \"msg\": \"e17\"}, {\"step\": 18, \"kind\": \"trace\", \"msg\": \"e18\"}, {\"step\": 19, \"kind\": \"trace\", \"msg\": \"e19\"}]}",
"original": "{\"request_id\": \"req-1\", \"events\": [{\"step\": 0, \"kind\": \"trace\", \"msg\": \"e0\"}, {\"step\": 1, \"kind\": \"trace\", \"msg\": \"e1\"}, {\"step\": 2, \"kind\": \"trace\", \"msg\": \"e2\"}, {\"step\": 3, \"kind\": \"trace\", \"msg\": \"e3\"}, {\"step\": 4, \"kind\": \"trace\", \"msg\": \"e4\"}, {\"step\": 5, \"kind\": \"trace\", \"msg\": \"e5\"}, {\"step\": 6, \"kind\": \"trace\", \"msg\": \"e6\"}, {\"step\": 7, \"kind\": \"trace\", \"msg\": \"e7\"}, {\"step\": 8, \"kind\": \"trace\", \"msg\": \"e8\"}, {\"step\": 9, \"kind\": \"trace\", \"msg\": \"e9\"}, {\"step\": 10, \"kind\": \"trace\", \"msg\": \"e10\"}, {\"step\": 11, \"kind\": \"trace\", \"msg\": \"e11\"}, {\"step\": 12, \"kind\": \"trace\", \"msg\": \"e12\"}, {\"step\": 13, \"kind\": \"trace\", \"msg\": \"e13\"}, {\"step\": 14, \"kind\": \"trace\", \"msg\": \"e14\"}, {\"step\": 15, \"kind\": \"trace\", \"msg\": \"e15\"}, {\"step\": 16, \"kind\": \"trace\", \"msg\": \"e16\"}, {\"step\": 17, \"kind\": \"trace\", \"msg\": \"e17\"}, {\"step\": 18, \"kind\": \"trace\", \"msg\": \"e18\"}, {\"step\": 19, \"kind\": \"trace\", \"msg\": \"e19\"}]}",
"strategy": "skip:unique_entities_no_signal(20->20)",
- "was_modified": true
+ "was_modified": false
},
- "recorded_at": "2026-04-27T06:59:39.935923+00:00",
+ "recorded_at": "2026-10-08T19:16:37.831265+00:00",
"transform": "smart_crusher"
}
diff --git a/tests/parity/fixtures/smart_crusher/nulls_and_bools_06d6fd6adcab.json b/tests/parity/fixtures/smart_crusher/nulls_and_bools_9f40c7fded19.json
similarity index 61%
rename from tests/parity/fixtures/smart_crusher/nulls_and_bools_06d6fd6adcab.json
rename to tests/parity/fixtures/smart_crusher/nulls_and_bools_9f40c7fded19.json
index 041adc374..a1b39b8b3 100644
--- a/tests/parity/fixtures/smart_crusher/nulls_and_bools_06d6fd6adcab.json
+++ b/tests/parity/fixtures/smart_crusher/nulls_and_bools_9f40c7fded19.json
@@ -1,11 +1,18 @@
{
"config": {
+ "compaction_core_field_fraction": 0.8,
+ "compaction_heterogeneous_core_ratio": 0.6,
+ "compaction_max_buckets": 8,
+ "compaction_max_flatten_inner_keys": 6,
+ "compaction_min_buckets": 2,
"dedup_identical_items": true,
"enabled": true,
"factor_out_constants": false,
"first_fraction": 0.3,
"include_summaries": false,
"last_fraction": 0.15,
+ "lossless_min_savings_ratio": 0.15,
+ "lossless_only": false,
"max_items_after_crush": 15,
"min_items_to_analyze": 5,
"min_tokens_to_crush": 200,
@@ -21,14 +28,14 @@
"content": "[null, true, false, null, true, false, null]",
"query": ""
},
- "input_sha256": "06d6fd6adcab251afc35de66e5cd54cfc21f15175bb082286cd1255b0f5bd965",
+ "input_sha256": "9f40c7fded195680cbe1281d4061246c142926ca3db3c703654a9a7182591c92",
"label": "nulls_and_bools",
"output": {
- "compressed": "[null,true,false,null,true,false,null]",
+ "compressed": "[null, true, false, null, true, false, null]",
"original": "[null, true, false, null, true, false, null]",
"strategy": "mixed:passthrough(7->7)",
- "was_modified": true
+ "was_modified": false
},
- "recorded_at": "2026-04-27T06:59:40.029745+00:00",
+ "recorded_at": "2026-10-09T03:59:17.344613+00:00",
"transform": "smart_crusher"
}
diff --git a/tests/parity/fixtures/smart_crusher/short_array_passthrough_aeb16224b61f.json b/tests/parity/fixtures/smart_crusher/short_array_passthrough_70abace7c488.json
similarity index 60%
rename from tests/parity/fixtures/smart_crusher/short_array_passthrough_aeb16224b61f.json
rename to tests/parity/fixtures/smart_crusher/short_array_passthrough_70abace7c488.json
index fb2fc97d5..3cc405de0 100644
--- a/tests/parity/fixtures/smart_crusher/short_array_passthrough_aeb16224b61f.json
+++ b/tests/parity/fixtures/smart_crusher/short_array_passthrough_70abace7c488.json
@@ -1,11 +1,18 @@
{
"config": {
+ "compaction_core_field_fraction": 0.8,
+ "compaction_heterogeneous_core_ratio": 0.6,
+ "compaction_max_buckets": 8,
+ "compaction_max_flatten_inner_keys": 6,
+ "compaction_min_buckets": 2,
"dedup_identical_items": true,
"enabled": true,
"factor_out_constants": false,
"first_fraction": 0.3,
"include_summaries": false,
"last_fraction": 0.15,
+ "lossless_min_savings_ratio": 0.15,
+ "lossless_only": false,
"max_items_after_crush": 15,
"min_items_to_analyze": 5,
"min_tokens_to_crush": 200,
@@ -21,14 +28,14 @@
"content": "[1, 2, 3]",
"query": ""
},
- "input_sha256": "aeb16224b61f335e6813afa323dc5d22c23b357fe6eb10ee2f8cfb3de6bfe4cf",
+ "input_sha256": "70abace7c488b4acb90c60cef985af22980bdd4d58dfb275140cc909c8a3d97f",
"label": "short_array_passthrough",
"output": {
- "compressed": "[1,2,3]",
+ "compressed": "[1, 2, 3]",
"original": "[1, 2, 3]",
"strategy": "passthrough",
- "was_modified": true
+ "was_modified": false
},
- "recorded_at": "2026-04-27T06:59:39.895735+00:00",
+ "recorded_at": "2026-10-09T03:59:17.333122+00:00",
"transform": "smart_crusher"
}
diff --git a/tests/parity/fixtures/smart_crusher/small_object_passthrough_801afa36a32c.json b/tests/parity/fixtures/smart_crusher/small_object_passthrough_7249ffeaddf5.json
similarity index 61%
rename from tests/parity/fixtures/smart_crusher/small_object_passthrough_801afa36a32c.json
rename to tests/parity/fixtures/smart_crusher/small_object_passthrough_7249ffeaddf5.json
index c5c6e03b3..d30c58303 100644
--- a/tests/parity/fixtures/smart_crusher/small_object_passthrough_801afa36a32c.json
+++ b/tests/parity/fixtures/smart_crusher/small_object_passthrough_7249ffeaddf5.json
@@ -1,11 +1,18 @@
{
"config": {
+ "compaction_core_field_fraction": 0.8,
+ "compaction_heterogeneous_core_ratio": 0.6,
+ "compaction_max_buckets": 8,
+ "compaction_max_flatten_inner_keys": 6,
+ "compaction_min_buckets": 2,
"dedup_identical_items": true,
"enabled": true,
"factor_out_constants": false,
"first_fraction": 0.3,
"include_summaries": false,
"last_fraction": 0.15,
+ "lossless_min_savings_ratio": 0.15,
+ "lossless_only": false,
"max_items_after_crush": 15,
"min_items_to_analyze": 5,
"min_tokens_to_crush": 200,
@@ -21,14 +28,14 @@
"content": "{\"a\": 1, \"b\": 2, \"c\": \"hello\"}",
"query": ""
},
- "input_sha256": "801afa36a32c7ed7066e80c8a28fefedbb979918e679b5640803fb5d06ef6fbf",
+ "input_sha256": "7249ffeaddf54726945ac4baf298cd33fb3bfb89f672983749065d7dfdf61553",
"label": "small_object_passthrough",
"output": {
- "compressed": "{\"a\":1,\"b\":2,\"c\":\"hello\"}",
+ "compressed": "{\"a\": 1, \"b\": 2, \"c\": \"hello\"}",
"original": "{\"a\": 1, \"b\": 2, \"c\": \"hello\"}",
"strategy": "passthrough",
- "was_modified": true
+ "was_modified": false
},
- "recorded_at": "2026-04-27T06:59:39.895485+00:00",
+ "recorded_at": "2026-10-09T03:59:17.315451+00:00",
"transform": "smart_crusher"
}
diff --git a/tests/parity/fixtures/smart_crusher/time_series_50_25cd28df5a50.json b/tests/parity/fixtures/smart_crusher/time_series_50_2c51eb9374ce.json
similarity index 66%
rename from tests/parity/fixtures/smart_crusher/time_series_50_25cd28df5a50.json
rename to tests/parity/fixtures/smart_crusher/time_series_50_2c51eb9374ce.json
index 05622ecde..14ff2d401 100644
--- a/tests/parity/fixtures/smart_crusher/time_series_50_25cd28df5a50.json
+++ b/tests/parity/fixtures/smart_crusher/time_series_50_2c51eb9374ce.json
@@ -1,11 +1,18 @@
{
"config": {
+ "compaction_core_field_fraction": 0.8,
+ "compaction_heterogeneous_core_ratio": 0.6,
+ "compaction_max_buckets": 8,
+ "compaction_max_flatten_inner_keys": 6,
+ "compaction_min_buckets": 2,
"dedup_identical_items": true,
"enabled": true,
"factor_out_constants": false,
"first_fraction": 0.3,
"include_summaries": false,
"last_fraction": 0.15,
+ "lossless_min_savings_ratio": 0.15,
+ "lossless_only": false,
"max_items_after_crush": 15,
"min_items_to_analyze": 5,
"min_tokens_to_crush": 200,
@@ -21,14 +28,14 @@
"content": "[{\"ts\": 1000, \"metric\": 0.0, \"host\": \"host-0\"}, {\"ts\": 1001, \"metric\": 1.5, \"host\": \"host-1\"}, {\"ts\": 1002, \"metric\": 3.0, \"host\": \"host-2\"}, {\"ts\": 1003, \"metric\": 4.5, \"host\": \"host-0\"}, {\"ts\": 1004, \"metric\": 6.0, \"host\": \"host-1\"}, {\"ts\": 1005, \"metric\": 7.5, \"host\": \"host-2\"}, {\"ts\": 1006, \"metric\": 9.0, \"host\": \"host-0\"}, {\"ts\": 1007, \"metric\": 10.5, \"host\": \"host-1\"}, {\"ts\": 1008, \"metric\": 12.0, \"host\": \"host-2\"}, {\"ts\": 1009, \"metric\": 13.5, \"host\": \"host-0\"}, {\"ts\": 1010, \"metric\": 15.0, \"host\": \"host-1\"}, {\"ts\": 1011, \"metric\": 16.5, \"host\": \"host-2\"}, {\"ts\": 1012, \"metric\": 18.0, \"host\": \"host-0\"}, {\"ts\": 1013, \"metric\": 19.5, \"host\": \"host-1\"}, {\"ts\": 1014, \"metric\": 21.0, \"host\": \"host-2\"}, {\"ts\": 1015, \"metric\": 22.5, \"host\": \"host-0\"}, {\"ts\": 1016, \"metric\": 24.0, \"host\": \"host-1\"}, {\"ts\": 1017, \"metric\": 25.5, \"host\": \"host-2\"}, {\"ts\": 1018, \"metric\": 27.0, \"host\": \"host-0\"}, {\"ts\": 1019, \"metric\": 28.5, \"host\": \"host-1\"}, {\"ts\": 1020, \"metric\": 30.0, \"host\": \"host-2\"}, {\"ts\": 1021, \"metric\": 31.5, \"host\": \"host-0\"}, {\"ts\": 1022, \"metric\": 33.0, \"host\": \"host-1\"}, {\"ts\": 1023, \"metric\": 34.5, \"host\": \"host-2\"}, {\"ts\": 1024, \"metric\": 36.0, \"host\": \"host-0\"}, {\"ts\": 1025, \"metric\": 37.5, \"host\": \"host-1\"}, {\"ts\": 1026, \"metric\": 39.0, \"host\": \"host-2\"}, {\"ts\": 1027, \"metric\": 40.5, \"host\": \"host-0\"}, {\"ts\": 1028, \"metric\": 42.0, \"host\": \"host-1\"}, {\"ts\": 1029, \"metric\": 43.5, \"host\": \"host-2\"}, {\"ts\": 1030, \"metric\": 45.0, \"host\": \"host-0\"}, {\"ts\": 1031, \"metric\": 46.5, \"host\": \"host-1\"}, {\"ts\": 1032, \"metric\": 48.0, \"host\": \"host-2\"}, {\"ts\": 1033, \"metric\": 49.5, \"host\": \"host-0\"}, {\"ts\": 1034, \"metric\": 51.0, \"host\": \"host-1\"}, {\"ts\": 1035, \"metric\": 52.5, \"host\": \"host-2\"}, {\"ts\": 1036, \"metric\": 54.0, \"host\": \"host-0\"}, {\"ts\": 1037, \"metric\": 55.5, \"host\": \"host-1\"}, {\"ts\": 1038, \"metric\": 57.0, \"host\": \"host-2\"}, {\"ts\": 1039, \"metric\": 58.5, \"host\": \"host-0\"}, {\"ts\": 1040, \"metric\": 60.0, \"host\": \"host-1\"}, {\"ts\": 1041, \"metric\": 61.5, \"host\": \"host-2\"}, {\"ts\": 1042, \"metric\": 63.0, \"host\": \"host-0\"}, {\"ts\": 1043, \"metric\": 64.5, \"host\": \"host-1\"}, {\"ts\": 1044, \"metric\": 66.0, \"host\": \"host-2\"}, {\"ts\": 1045, \"metric\": 67.5, \"host\": \"host-0\"}, {\"ts\": 1046, \"metric\": 69.0, \"host\": \"host-1\"}, {\"ts\": 1047, \"metric\": 70.5, \"host\": \"host-2\"}, {\"ts\": 1048, \"metric\": 72.0, \"host\": \"host-0\"}, {\"ts\": 1049, \"metric\": 73.5, \"host\": \"host-1\"}]",
"query": ""
},
- "input_sha256": "25cd28df5a50521ac1f669482e1fe30bf2422f555ddd1a4d7506ca686dbd0734",
+ "input_sha256": "2c51eb9374cee91458257eddb1185f23ff9890cfe8c21bb07ba88d6ba1ae0d00",
"label": "time_series_50",
"output": {
- "compressed": "[{\"ts\":1000,\"metric\":0.0,\"host\":\"host-0\"},{\"ts\":1001,\"metric\":1.5,\"host\":\"host-1\"},{\"ts\":1002,\"metric\":3.0,\"host\":\"host-2\"},{\"ts\":1003,\"metric\":4.5,\"host\":\"host-0\"},{\"ts\":1004,\"metric\":6.0,\"host\":\"host-1\"},{\"ts\":1005,\"metric\":7.5,\"host\":\"host-2\"},{\"ts\":1006,\"metric\":9.0,\"host\":\"host-0\"},{\"ts\":1007,\"metric\":10.5,\"host\":\"host-1\"},{\"ts\":1008,\"metric\":12.0,\"host\":\"host-2\"},{\"ts\":1009,\"metric\":13.5,\"host\":\"host-0\"},{\"ts\":1010,\"metric\":15.0,\"host\":\"host-1\"},{\"ts\":1011,\"metric\":16.5,\"host\":\"host-2\"},{\"ts\":1012,\"metric\":18.0,\"host\":\"host-0\"},{\"ts\":1013,\"metric\":19.5,\"host\":\"host-1\"},{\"ts\":1014,\"metric\":21.0,\"host\":\"host-2\"},{\"ts\":1015,\"metric\":22.5,\"host\":\"host-0\"},{\"ts\":1016,\"metric\":24.0,\"host\":\"host-1\"},{\"ts\":1017,\"metric\":25.5,\"host\":\"host-2\"},{\"ts\":1018,\"metric\":27.0,\"host\":\"host-0\"},{\"ts\":1019,\"metric\":28.5,\"host\":\"host-1\"},{\"ts\":1020,\"metric\":30.0,\"host\":\"host-2\"},{\"ts\":1021,\"metric\":31.5,\"host\":\"host-0\"},{\"ts\":1022,\"metric\":33.0,\"host\":\"host-1\"},{\"ts\":1023,\"metric\":34.5,\"host\":\"host-2\"},{\"ts\":1024,\"metric\":36.0,\"host\":\"host-0\"},{\"ts\":1025,\"metric\":37.5,\"host\":\"host-1\"},{\"ts\":1026,\"metric\":39.0,\"host\":\"host-2\"},{\"ts\":1027,\"metric\":40.5,\"host\":\"host-0\"},{\"ts\":1028,\"metric\":42.0,\"host\":\"host-1\"},{\"ts\":1029,\"metric\":43.5,\"host\":\"host-2\"},{\"ts\":1030,\"metric\":45.0,\"host\":\"host-0\"},{\"ts\":1031,\"metric\":46.5,\"host\":\"host-1\"},{\"ts\":1032,\"metric\":48.0,\"host\":\"host-2\"},{\"ts\":1033,\"metric\":49.5,\"host\":\"host-0\"},{\"ts\":1034,\"metric\":51.0,\"host\":\"host-1\"},{\"ts\":1035,\"metric\":52.5,\"host\":\"host-2\"},{\"ts\":1036,\"metric\":54.0,\"host\":\"host-0\"},{\"ts\":1037,\"metric\":55.5,\"host\":\"host-1\"},{\"ts\":1038,\"metric\":57.0,\"host\":\"host-2\"},{\"ts\":1039,\"metric\":58.5,\"host\":\"host-0\"},{\"ts\":1040,\"metric\":60.0,\"host\":\"host-1\"},{\"ts\":1041,\"metric\":61.5,\"host\":\"host-2\"},{\"ts\":1042,\"metric\":63.0,\"host\":\"host-0\"},{\"ts\":1043,\"metric\":64.5,\"host\":\"host-1\"},{\"ts\":1044,\"metric\":66.0,\"host\":\"host-2\"},{\"ts\":1045,\"metric\":67.5,\"host\":\"host-0\"},{\"ts\":1046,\"metric\":69.0,\"host\":\"host-1\"},{\"ts\":1047,\"metric\":70.5,\"host\":\"host-2\"},{\"ts\":1048,\"metric\":72.0,\"host\":\"host-0\"},{\"ts\":1049,\"metric\":73.5,\"host\":\"host-1\"}]",
+ "compressed": "[{\"ts\": 1000, \"metric\": 0.0, \"host\": \"host-0\"}, {\"ts\": 1001, \"metric\": 1.5, \"host\": \"host-1\"}, {\"ts\": 1002, \"metric\": 3.0, \"host\": \"host-2\"}, {\"ts\": 1003, \"metric\": 4.5, \"host\": \"host-0\"}, {\"ts\": 1004, \"metric\": 6.0, \"host\": \"host-1\"}, {\"ts\": 1005, \"metric\": 7.5, \"host\": \"host-2\"}, {\"ts\": 1006, \"metric\": 9.0, \"host\": \"host-0\"}, {\"ts\": 1007, \"metric\": 10.5, \"host\": \"host-1\"}, {\"ts\": 1008, \"metric\": 12.0, \"host\": \"host-2\"}, {\"ts\": 1009, \"metric\": 13.5, \"host\": \"host-0\"}, {\"ts\": 1010, \"metric\": 15.0, \"host\": \"host-1\"}, {\"ts\": 1011, \"metric\": 16.5, \"host\": \"host-2\"}, {\"ts\": 1012, \"metric\": 18.0, \"host\": \"host-0\"}, {\"ts\": 1013, \"metric\": 19.5, \"host\": \"host-1\"}, {\"ts\": 1014, \"metric\": 21.0, \"host\": \"host-2\"}, {\"ts\": 1015, \"metric\": 22.5, \"host\": \"host-0\"}, {\"ts\": 1016, \"metric\": 24.0, \"host\": \"host-1\"}, {\"ts\": 1017, \"metric\": 25.5, \"host\": \"host-2\"}, {\"ts\": 1018, \"metric\": 27.0, \"host\": \"host-0\"}, {\"ts\": 1019, \"metric\": 28.5, \"host\": \"host-1\"}, {\"ts\": 1020, \"metric\": 30.0, \"host\": \"host-2\"}, {\"ts\": 1021, \"metric\": 31.5, \"host\": \"host-0\"}, {\"ts\": 1022, \"metric\": 33.0, \"host\": \"host-1\"}, {\"ts\": 1023, \"metric\": 34.5, \"host\": \"host-2\"}, {\"ts\": 1024, \"metric\": 36.0, \"host\": \"host-0\"}, {\"ts\": 1025, \"metric\": 37.5, \"host\": \"host-1\"}, {\"ts\": 1026, \"metric\": 39.0, \"host\": \"host-2\"}, {\"ts\": 1027, \"metric\": 40.5, \"host\": \"host-0\"}, {\"ts\": 1028, \"metric\": 42.0, \"host\": \"host-1\"}, {\"ts\": 1029, \"metric\": 43.5, \"host\": \"host-2\"}, {\"ts\": 1030, \"metric\": 45.0, \"host\": \"host-0\"}, {\"ts\": 1031, \"metric\": 46.5, \"host\": \"host-1\"}, {\"ts\": 1032, \"metric\": 48.0, \"host\": \"host-2\"}, {\"ts\": 1033, \"metric\": 49.5, \"host\": \"host-0\"}, {\"ts\": 1034, \"metric\": 51.0, \"host\": \"host-1\"}, {\"ts\": 1035, \"metric\": 52.5, \"host\": \"host-2\"}, {\"ts\": 1036, \"metric\": 54.0, \"host\": \"host-0\"}, {\"ts\": 1037, \"metric\": 55.5, \"host\": \"host-1\"}, {\"ts\": 1038, \"metric\": 57.0, \"host\": \"host-2\"}, {\"ts\": 1039, \"metric\": 58.5, \"host\": \"host-0\"}, {\"ts\": 1040, \"metric\": 60.0, \"host\": \"host-1\"}, {\"ts\": 1041, \"metric\": 61.5, \"host\": \"host-2\"}, {\"ts\": 1042, \"metric\": 63.0, \"host\": \"host-0\"}, {\"ts\": 1043, \"metric\": 64.5, \"host\": \"host-1\"}, {\"ts\": 1044, \"metric\": 66.0, \"host\": \"host-2\"}, {\"ts\": 1045, \"metric\": 67.5, \"host\": \"host-0\"}, {\"ts\": 1046, \"metric\": 69.0, \"host\": \"host-1\"}, {\"ts\": 1047, \"metric\": 70.5, \"host\": \"host-2\"}, {\"ts\": 1048, \"metric\": 72.0, \"host\": \"host-0\"}, {\"ts\": 1049, \"metric\": 73.5, \"host\": \"host-1\"}]",
"original": "[{\"ts\": 1000, \"metric\": 0.0, \"host\": \"host-0\"}, {\"ts\": 1001, \"metric\": 1.5, \"host\": \"host-1\"}, {\"ts\": 1002, \"metric\": 3.0, \"host\": \"host-2\"}, {\"ts\": 1003, \"metric\": 4.5, \"host\": \"host-0\"}, {\"ts\": 1004, \"metric\": 6.0, \"host\": \"host-1\"}, {\"ts\": 1005, \"metric\": 7.5, \"host\": \"host-2\"}, {\"ts\": 1006, \"metric\": 9.0, \"host\": \"host-0\"}, {\"ts\": 1007, \"metric\": 10.5, \"host\": \"host-1\"}, {\"ts\": 1008, \"metric\": 12.0, \"host\": \"host-2\"}, {\"ts\": 1009, \"metric\": 13.5, \"host\": \"host-0\"}, {\"ts\": 1010, \"metric\": 15.0, \"host\": \"host-1\"}, {\"ts\": 1011, \"metric\": 16.5, \"host\": \"host-2\"}, {\"ts\": 1012, \"metric\": 18.0, \"host\": \"host-0\"}, {\"ts\": 1013, \"metric\": 19.5, \"host\": \"host-1\"}, {\"ts\": 1014, \"metric\": 21.0, \"host\": \"host-2\"}, {\"ts\": 1015, \"metric\": 22.5, \"host\": \"host-0\"}, {\"ts\": 1016, \"metric\": 24.0, \"host\": \"host-1\"}, {\"ts\": 1017, \"metric\": 25.5, \"host\": \"host-2\"}, {\"ts\": 1018, \"metric\": 27.0, \"host\": \"host-0\"}, {\"ts\": 1019, \"metric\": 28.5, \"host\": \"host-1\"}, {\"ts\": 1020, \"metric\": 30.0, \"host\": \"host-2\"}, {\"ts\": 1021, \"metric\": 31.5, \"host\": \"host-0\"}, {\"ts\": 1022, \"metric\": 33.0, \"host\": \"host-1\"}, {\"ts\": 1023, \"metric\": 34.5, \"host\": \"host-2\"}, {\"ts\": 1024, \"metric\": 36.0, \"host\": \"host-0\"}, {\"ts\": 1025, \"metric\": 37.5, \"host\": \"host-1\"}, {\"ts\": 1026, \"metric\": 39.0, \"host\": \"host-2\"}, {\"ts\": 1027, \"metric\": 40.5, \"host\": \"host-0\"}, {\"ts\": 1028, \"metric\": 42.0, \"host\": \"host-1\"}, {\"ts\": 1029, \"metric\": 43.5, \"host\": \"host-2\"}, {\"ts\": 1030, \"metric\": 45.0, \"host\": \"host-0\"}, {\"ts\": 1031, \"metric\": 46.5, \"host\": \"host-1\"}, {\"ts\": 1032, \"metric\": 48.0, \"host\": \"host-2\"}, {\"ts\": 1033, \"metric\": 49.5, \"host\": \"host-0\"}, {\"ts\": 1034, \"metric\": 51.0, \"host\": \"host-1\"}, {\"ts\": 1035, \"metric\": 52.5, \"host\": \"host-2\"}, {\"ts\": 1036, \"metric\": 54.0, \"host\": \"host-0\"}, {\"ts\": 1037, \"metric\": 55.5, \"host\": \"host-1\"}, {\"ts\": 1038, \"metric\": 57.0, \"host\": \"host-2\"}, {\"ts\": 1039, \"metric\": 58.5, \"host\": \"host-0\"}, {\"ts\": 1040, \"metric\": 60.0, \"host\": \"host-1\"}, {\"ts\": 1041, \"metric\": 61.5, \"host\": \"host-2\"}, {\"ts\": 1042, \"metric\": 63.0, \"host\": \"host-0\"}, {\"ts\": 1043, \"metric\": 64.5, \"host\": \"host-1\"}, {\"ts\": 1044, \"metric\": 66.0, \"host\": \"host-2\"}, {\"ts\": 1045, \"metric\": 67.5, \"host\": \"host-0\"}, {\"ts\": 1046, \"metric\": 69.0, \"host\": \"host-1\"}, {\"ts\": 1047, \"metric\": 70.5, \"host\": \"host-2\"}, {\"ts\": 1048, \"metric\": 72.0, \"host\": \"host-0\"}, {\"ts\": 1049, \"metric\": 73.5, \"host\": \"host-1\"}]",
"strategy": "skip:unique_entities_no_signal(50->50)",
- "was_modified": true
+ "was_modified": false
},
- "recorded_at": "2026-04-27T06:59:40.020402+00:00",
+ "recorded_at": "2026-10-08T19:16:37.953147+00:00",
"transform": "smart_crusher"
}
diff --git a/tests/parity/fixtures/smart_crusher/unicode_dict_array_4820ed6c0c9a.json b/tests/parity/fixtures/smart_crusher/unicode_dict_array_b85509fce501.json
similarity index 66%
rename from tests/parity/fixtures/smart_crusher/unicode_dict_array_4820ed6c0c9a.json
rename to tests/parity/fixtures/smart_crusher/unicode_dict_array_b85509fce501.json
index a4af718d2..273f0caa7 100644
--- a/tests/parity/fixtures/smart_crusher/unicode_dict_array_4820ed6c0c9a.json
+++ b/tests/parity/fixtures/smart_crusher/unicode_dict_array_b85509fce501.json
@@ -1,11 +1,18 @@
{
"config": {
+ "compaction_core_field_fraction": 0.8,
+ "compaction_heterogeneous_core_ratio": 0.6,
+ "compaction_max_buckets": 8,
+ "compaction_max_flatten_inner_keys": 6,
+ "compaction_min_buckets": 2,
"dedup_identical_items": true,
"enabled": true,
"factor_out_constants": false,
"first_fraction": 0.3,
"include_summaries": false,
"last_fraction": 0.15,
+ "lossless_min_savings_ratio": 0.15,
+ "lossless_only": false,
"max_items_after_crush": 15,
"min_items_to_analyze": 5,
"min_tokens_to_crush": 200,
@@ -21,14 +28,14 @@
"content": "[{\"id\": 0, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 0\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 1, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 1\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 2, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 2\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 3, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 3\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 4, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 4\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 5, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 5\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 6, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 6\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 7, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 7\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 8, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 8\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 9, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 9\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 10, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 10\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 11, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 11\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 12, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 12\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 13, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 13\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 14, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 14\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 15, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 15\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 16, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 16\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 17, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 17\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 18, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 18\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 19, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 19\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}]",
"query": ""
},
- "input_sha256": "4820ed6c0c9a4f52ab17d9bd0d6e7010fea978a227b24cb2a7b0cdba5f6fe5a6",
+ "input_sha256": "b85509fce50161c9b5f2e0892bfc8f6caab8e40df68cd3c6a136808d03b4cb0f",
"label": "unicode_dict_array",
"output": {
- "compressed": "[{\"id\":0,\"msg\":\"hello \u4e2d\u6587 \u0440\u0443\u0441\u0441\u043a\u0438\u0439 0\",\"tag\":\"\u0442\u0435\u0441\u0442\"},{\"id\":1,\"msg\":\"hello \u4e2d\u6587 \u0440\u0443\u0441\u0441\u043a\u0438\u0439 1\",\"tag\":\"\u0442\u0435\u0441\u0442\"},{\"id\":2,\"msg\":\"hello \u4e2d\u6587 \u0440\u0443\u0441\u0441\u043a\u0438\u0439 2\",\"tag\":\"\u0442\u0435\u0441\u0442\"},{\"id\":3,\"msg\":\"hello \u4e2d\u6587 \u0440\u0443\u0441\u0441\u043a\u0438\u0439 3\",\"tag\":\"\u0442\u0435\u0441\u0442\"},{\"id\":4,\"msg\":\"hello \u4e2d\u6587 \u0440\u0443\u0441\u0441\u043a\u0438\u0439 4\",\"tag\":\"\u0442\u0435\u0441\u0442\"},{\"id\":5,\"msg\":\"hello \u4e2d\u6587 \u0440\u0443\u0441\u0441\u043a\u0438\u0439 5\",\"tag\":\"\u0442\u0435\u0441\u0442\"},{\"id\":6,\"msg\":\"hello \u4e2d\u6587 \u0440\u0443\u0441\u0441\u043a\u0438\u0439 6\",\"tag\":\"\u0442\u0435\u0441\u0442\"},{\"id\":7,\"msg\":\"hello \u4e2d\u6587 \u0440\u0443\u0441\u0441\u043a\u0438\u0439 7\",\"tag\":\"\u0442\u0435\u0441\u0442\"},{\"id\":8,\"msg\":\"hello \u4e2d\u6587 \u0440\u0443\u0441\u0441\u043a\u0438\u0439 8\",\"tag\":\"\u0442\u0435\u0441\u0442\"},{\"id\":9,\"msg\":\"hello \u4e2d\u6587 \u0440\u0443\u0441\u0441\u043a\u0438\u0439 9\",\"tag\":\"\u0442\u0435\u0441\u0442\"},{\"id\":10,\"msg\":\"hello \u4e2d\u6587 \u0440\u0443\u0441\u0441\u043a\u0438\u0439 10\",\"tag\":\"\u0442\u0435\u0441\u0442\"},{\"id\":11,\"msg\":\"hello \u4e2d\u6587 \u0440\u0443\u0441\u0441\u043a\u0438\u0439 11\",\"tag\":\"\u0442\u0435\u0441\u0442\"},{\"id\":12,\"msg\":\"hello \u4e2d\u6587 \u0440\u0443\u0441\u0441\u043a\u0438\u0439 12\",\"tag\":\"\u0442\u0435\u0441\u0442\"},{\"id\":13,\"msg\":\"hello \u4e2d\u6587 \u0440\u0443\u0441\u0441\u043a\u0438\u0439 13\",\"tag\":\"\u0442\u0435\u0441\u0442\"},{\"id\":14,\"msg\":\"hello \u4e2d\u6587 \u0440\u0443\u0441\u0441\u043a\u0438\u0439 14\",\"tag\":\"\u0442\u0435\u0441\u0442\"},{\"id\":15,\"msg\":\"hello \u4e2d\u6587 \u0440\u0443\u0441\u0441\u043a\u0438\u0439 15\",\"tag\":\"\u0442\u0435\u0441\u0442\"},{\"id\":16,\"msg\":\"hello \u4e2d\u6587 \u0440\u0443\u0441\u0441\u043a\u0438\u0439 16\",\"tag\":\"\u0442\u0435\u0441\u0442\"},{\"id\":17,\"msg\":\"hello \u4e2d\u6587 \u0440\u0443\u0441\u0441\u043a\u0438\u0439 17\",\"tag\":\"\u0442\u0435\u0441\u0442\"},{\"id\":18,\"msg\":\"hello \u4e2d\u6587 \u0440\u0443\u0441\u0441\u043a\u0438\u0439 18\",\"tag\":\"\u0442\u0435\u0441\u0442\"},{\"id\":19,\"msg\":\"hello \u4e2d\u6587 \u0440\u0443\u0441\u0441\u043a\u0438\u0439 19\",\"tag\":\"\u0442\u0435\u0441\u0442\"}]",
+ "compressed": "[{\"id\": 0, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 0\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 1, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 1\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 2, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 2\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 3, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 3\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 4, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 4\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 5, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 5\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 6, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 6\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 7, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 7\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 8, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 8\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 9, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 9\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 10, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 10\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 11, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 11\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 12, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 12\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 13, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 13\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 14, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 14\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 15, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 15\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 16, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 16\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 17, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 17\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 18, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 18\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 19, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 19\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}]",
"original": "[{\"id\": 0, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 0\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 1, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 1\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 2, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 2\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 3, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 3\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 4, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 4\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 5, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 5\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 6, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 6\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 7, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 7\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 8, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 8\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 9, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 9\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 10, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 10\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 11, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 11\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 12, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 12\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 13, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 13\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 14, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 14\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 15, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 15\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 16, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 16\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 17, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 17\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 18, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 18\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}, {\"id\": 19, \"msg\": \"hello \\u4e2d\\u6587 \\u0440\\u0443\\u0441\\u0441\\u043a\\u0438\\u0439 19\", \"tag\": \"\\u0442\\u0435\\u0441\\u0442\"}]",
"strategy": "skip:unique_entities_no_signal(20->20)",
- "was_modified": true
+ "was_modified": false
},
- "recorded_at": "2026-04-27T06:59:39.970946+00:00",
+ "recorded_at": "2026-10-08T19:16:37.874112+00:00",
"transform": "smart_crusher"
}
diff --git a/tests/parity/record_smart_crusher.py b/tests/parity/record_smart_crusher.py
index e8e24a7fe..3e5549479 100644
--- a/tests/parity/record_smart_crusher.py
+++ b/tests/parity/record_smart_crusher.py
@@ -33,11 +33,13 @@ from __future__ import annotations
import datetime as _dt
import hashlib
+import inspect
import json
from dataclasses import asdict
from pathlib import Path
from typing import Any
+from headroom._core import SmartCrusherConfig as NativeSmartCrusherConfig
from headroom.transforms.smart_crusher import SmartCrusher, SmartCrusherConfig
_REPO_ROOT = Path(__file__).resolve().parent.parent.parent
@@ -57,11 +59,14 @@ def _record(
config: SmartCrusherConfig | None = None,
) -> Path:
cfg = config or SmartCrusherConfig()
- crusher = SmartCrusher(config=cfg)
+ # Match both parity comparators' legacy keep/drop mode.
+ crusher = SmartCrusher(config=cfg, with_compaction=False)
result = crusher.crush(content, query=query, bias=bias)
payload_input = {"content": content, "query": query, "bias": bias}
- payload_config = asdict(cfg)
+ # Fixtures are replayed through PyO3, not the wrapper-only guard options.
+ native_parameters = inspect.signature(NativeSmartCrusherConfig).parameters
+ payload_config = {key: value for key, value in asdict(cfg).items() if key in native_parameters}
payload_output = {
"compressed": result.compressed,
"original": result.original,
@@ -89,7 +94,7 @@ def _record(
_FIXTURES_DIR.mkdir(parents=True, exist_ok=True)
target = _FIXTURES_DIR / f"{label}_{digest[:12]}.json"
- target.write_text(json.dumps(fixture, indent=2, sort_keys=True) + "\n")
+ target.write_text(json.dumps(fixture, indent=2, sort_keys=True) + "\n", newline="\n")
return target
@@ -103,25 +108,11 @@ def _scenarios() -> list[tuple[str, str, str, float]]:
# unchanged; trivially byte-equal.
out.append(("non_json_passthrough", "this is not json at all", "", 1.0))
- # 2. JSON object with no array fields long enough to crush.
- out.append(
- (
- "small_object_passthrough",
- json.dumps({"a": 1, "b": 2, "c": "hello"}),
- "",
- 1.0,
- )
- )
+ # 2. An object without crushable arrays retains its whitespace.
+ out.append(("small_object_passthrough", json.dumps({"a": 1, "b": 2, "c": "hello"}), "", 1.0))
- # 3. Short array (below min_items_to_analyze=5) → passthrough.
- out.append(
- (
- "short_array_passthrough",
- json.dumps([1, 2, 3]),
- "",
- 1.0,
- )
- )
+ # 3. An array below the analysis threshold is byte-identical.
+ out.append(("short_array_passthrough", json.dumps([1, 2, 3]), "", 1.0))
# 4. Dict array with 30 items, varied integer status field.
# Exercises crush_array's adaptive_k → smart_sample / top_n path.
@@ -142,7 +133,8 @@ def _scenarios() -> list[tuple[str, str, str, float]]:
mixed_arr = ["start"] + list(range(20)) + ["middle"] + ["end"] * 5
out.append(("mixed_array", json.dumps(mixed_arr), "", 1.0))
- # 8. Nested: top-level dict whose `events` field is a long dict array.
+ # Nested arrays whose analysis keeps every item must preserve their
+ # enclosing object's original bytes rather than reserialize it.
nested = {
"request_id": "req-1",
"events": [{"step": i, "kind": "trace", "msg": f"e{i}"} for i in range(20)],
@@ -155,9 +147,7 @@ def _scenarios() -> list[tuple[str, str, str, float]]:
# 10. Bias < 1 (keep fewer) on the 30-dict case.
out.append(("dict_array_30_bias_low", json.dumps(items_30_dict), "", 0.7))
- # 11. Unicode payload — exercises the `ensure_ascii=False` path in
- # Python's safe_json_dumps. Rust's python_safe_json_dumps must emit
- # raw UTF-8 bytes here, not `\uXXXX` escapes.
+ # Preserve escaped Unicode in unchanged input, including whitespace.
unicode_items = [{"id": i, "msg": f"hello 中文 русский {i}", "tag": "тест"} for i in range(20)]
out.append(("unicode_dict_array", json.dumps(unicode_items), "", 1.0))
@@ -169,7 +159,8 @@ def _scenarios() -> list[tuple[str, str, str, float]]:
]
out.append(("dict_array_100_sequential", json.dumps(big_seq), "", 1.0))
- # 13. Time-series-like payload: monotonic timestamp + float metric.
+ # Time-series analysis may keep every point; its float spellings and
+ # original JSON formatting must then survive the Python/native bridge.
ts = [{"ts": 1000 + i, "metric": float(i * 1.5), "host": f"host-{i % 3}"} for i in range(50)]
out.append(("time_series_50", json.dumps(ts), "", 1.0))
@@ -180,18 +171,12 @@ def _scenarios() -> list[tuple[str, str, str, float]]:
# 15. Empty array — boundary case, must round-trip cleanly.
out.append(("empty_array", json.dumps([]), "", 1.0))
- # 16. Array of nulls and bools — non-crushable mixed type.
+ # 16. Non-crushable null/bool mixtures retain their original separators.
out.append(
- (
- "nulls_and_bools",
- json.dumps([None, True, False, None, True, False, None]),
- "",
- 1.0,
- )
+ ("nulls_and_bools", json.dumps([None, True, False, None, True, False, None]), "", 1.0)
)
- # 17. Deeply nested structure: 3-level depth with arrays at each
- # level. Exercises process_value's recursion.
+ # Recursive no-op analysis must preserve bytes through every object level.
deep = {"a": {"b": {"events": [{"i": i, "kind": "deep", "v": f"x{i}"} for i in range(15)]}}}
out.append(("nested_3deep_with_array", json.dumps(deep), "", 1.0))
diff --git a/tests/test_code_aware_regressions.py b/tests/test_code_aware_regressions.py
index 2dbcc2c11..65e8ee901 100644
--- a/tests/test_code_aware_regressions.py
+++ b/tests/test_code_aware_regressions.py
@@ -114,7 +114,7 @@ def test_content_router_code_aware_token_accounting_matches_router_estimate() ->
router = ContentRouter(ContentRouterConfig(enable_code_aware=True))
router._code_compressor = _FakeCodeCompressor(compressed_text)
- compressed, compressed_tokens, strategy_chain = router._apply_strategy_to_content(
+ compressed, compressed_tokens, strategy_chain, _reversible = router._apply_strategy_to_content(
original, CompressionStrategy.CODE_AWARE, context=""
)
diff --git a/tests/test_integrations/langchain/test_langchain_1x_compat.py b/tests/test_integrations/langchain/test_langchain_1x_compat.py
index a294a0333..43d81f8c1 100644
--- a/tests/test_integrations/langchain/test_langchain_1x_compat.py
+++ b/tests/test_integrations/langchain/test_langchain_1x_compat.py
@@ -33,16 +33,28 @@ except ImportError:
pytestmark = pytest.mark.skipif(not LANGCHAIN_AVAILABLE, reason="LangChain not installed")
+# Repeated rows require a real transformation, not serialization-only savings.
BIG_RESULT = json.dumps(
{
"results": [
- {"id": i, "user": f"user{i}", "plan": "pro", "status": "active"} for i in range(300)
+ {"id": i % 10, "user": f"user{i % 10}", "plan": "pro", "status": "active"}
+ for i in range(300)
],
"total": 300,
}
)
+def _assert_compressed_rows(output: str) -> None:
+ original = json.loads(BIG_RESULT)
+ compressed = json.loads(output)
+ kept = [row for row in compressed["results"] if "id" in row]
+ assert compressed["total"] == original["total"]
+ assert {row["id"]: row for row in kept} == {row["id"]: row for row in original["results"]}
+ assert len(kept) < len(original["results"])
+ assert len(output) < len(BIG_RESULT)
+
+
@tool
def query_database(query: str) -> str:
"""Query the users database. Returns JSON rows."""
@@ -110,8 +122,7 @@ class TestWrappedToolIsUsable:
wrapped = wrap_tools_with_headroom([query_database], min_chars_to_compress=1000)[0]
out = wrapped.invoke({"query": "signups"})
- assert isinstance(out, str) and out
- assert len(out) < len(BIG_RESULT)
+ _assert_compressed_rows(out)
def test_argument_schema_is_preserved(self):
from headroom.integrations import wrap_tools_with_headroom
@@ -129,7 +140,7 @@ class TestWrappedToolIsUsable:
wrapped = wrap_tools_with_headroom([query_database], min_chars_to_compress=1000)[0]
out = asyncio.run(wrapped.ainvoke({"query": "signups"}))
- assert len(out) < len(BIG_RESULT)
+ _assert_compressed_rows(out)
def test_unusable_args_schema_falls_back_to_inference(self):
"""A tool carrying a schema LangChain cannot use must still wrap."""
diff --git a/tests/test_lossless_mode.py b/tests/test_lossless_mode.py
index ba6c80e45..7de1265ac 100644
--- a/tests/test_lossless_mode.py
+++ b/tests/test_lossless_mode.py
@@ -252,7 +252,7 @@ def test_router_lossless_log_strategy_no_marker_and_recoverable() -> None:
+ "connection refused by upstream service alpha\n" * 40
+ "[info] boot sequence complete\n"
)
- out, _tokens, chain = router._apply_strategy_to_content(
+ out, _tokens, chain, _reversible = router._apply_strategy_to_content(
log, CompressionStrategy.LOG, context=""
)
_assert_no_marker(out)
@@ -273,7 +273,7 @@ def test_router_lossless_diff_strategy_no_marker() -> None:
"-old\n"
"+new\n"
)
- out, _tokens, chain = router._apply_strategy_to_content(
+ out, _tokens, chain, _reversible = router._apply_strategy_to_content(
diff, CompressionStrategy.DIFF, context=""
)
_assert_no_marker(out)
diff --git a/tests/test_openai_responses_context_compaction.py b/tests/test_openai_responses_context_compaction.py
index 16b2f8741..059a1801f 100644
--- a/tests/test_openai_responses_context_compaction.py
+++ b/tests/test_openai_responses_context_compaction.py
@@ -355,7 +355,7 @@ def test_content_router_retries_kompress_when_structured_strategy_noops(monkeypa
lambda value, context, question=None: ("short summary", 2),
)
- compressed, compressed_tokens, strategy_chain = router._apply_strategy_to_content(
+ compressed, compressed_tokens, strategy_chain, _reversible = router._apply_strategy_to_content(
content,
CompressionStrategy.SMART_CRUSHER,
context="",
diff --git a/tests/test_proxy_ccr.py b/tests/test_proxy_ccr.py
index 4cb122815..ba3307081 100644
--- a/tests/test_proxy_ccr.py
+++ b/tests/test_proxy_ccr.py
@@ -363,15 +363,14 @@ class TestEndToEndTOINIntegration:
from headroom.telemetry import ToolSignature
from headroom.transforms import SmartCrusher, TransformPipeline
- # Create tool output with 100 items that will trigger compression
- # Key: score field with varying values signals sortable data
- # Having repetitive category values helps trigger compression
+ # Duplicate ranked rows require real keep/drop compression, so
+ # recording TOIN and caching originals cannot pass on JSON formatting.
items = [
{
- "id": i,
- "score": 1000 - i, # Decreasing scores signal sorting
- "category": f"cat_{i % 3}", # Only 3 unique categories
- "status": "active" if i % 2 == 0 else "inactive", # Binary status
+ "id": i % 5,
+ "score": 1000 - (i % 5),
+ "category": f"cat_{i % 5}",
+ "status": "active",
}
for i in range(100)
]
@@ -434,40 +433,26 @@ class TestEndToEndTOINIntegration:
model_limit=200000,
)
- # Verify SmartCrusher was invoked (transform name starts with smart_crush)
- smart_crush_applied = any(
- t.startswith("smart_crush") or t.startswith("smart:") for t in result.transforms_applied
- )
- assert smart_crush_applied, (
- f"SmartCrusher should be in transforms: {result.transforms_applied}"
- )
+ output = result.messages[2]["content"][0]["content"]
+ # The pipeline appends its retrieval notice after the JSON payload.
+ compressed, _ = json.JSONDecoder().raw_decode(output)
+ kept = [row for row in compressed if "id" in row]
+ assert len(kept) < len(items)
+ assert all(row in items for row in kept)
- # Check if compression was actually performed (not skipped)
- # Skip messages look like "smart:skip:reason(100->100)"
- compression_was_skipped = any(
- "skip" in t.lower() for t in result.transforms_applied if "smart:" in t.lower()
- )
+ store = get_compression_store()
+ marker_text = compressed[-1]["_ccr_dropped"]
+ ccr_hash = marker_text[len("<<ccr:") :].split(" ", 1)[0]
+ entry = store.retrieve(ccr_hash)
+ assert entry is not None, "The rendered CCR marker must be retrievable"
+ assert json.loads(entry.original_content) == items
- # If compression happened, verify TOIN and store
- if not compression_was_skipped:
- # Verify compression store has the entry
- store = get_compression_store()
- stats = store.get_stats()
- assert stats["entry_count"] >= 1, "Should have cached entry"
-
- # Verify TOIN recorded the compression
- signature = ToolSignature.from_items(items)
- pattern = fresh_toin._patterns.get(signature.structure_hash)
- assert pattern is not None, (
- "TOIN should have recorded compression event. "
- "If this fails, SmartCrusher is not calling TOIN.record_compression."
- )
- assert pattern.total_compressions >= 1, "Should have at least 1 compression"
- else:
- # Compression was skipped - this is expected for some data patterns
- # The important thing is that SmartCrusher was invoked and made a decision
- # The other tests verify the full loop when compression does happen
- pass
+ signature = ToolSignature.from_items(items)
+ pattern = fresh_toin.get_pattern(signature.structure_hash)
+ assert pattern is not None, "TOIN should have recorded the real compression"
+ assert pattern.total_compressions == 1
+ assert pattern.total_items_seen == len(items)
+ assert pattern.total_items_kept == len(kept)
def test_retrieval_through_proxy_updates_toin_field_semantics(
self, fresh_toin, client_with_optimization
diff --git a/tests/test_proxy_compress_endpoint.py b/tests/test_proxy_compress_endpoint.py
index b6456497b..1a99fb75c 100644
--- a/tests/test_proxy_compress_endpoint.py
+++ b/tests/test_proxy_compress_endpoint.py
@@ -437,7 +437,7 @@ class TestCompressEndpointLossyInlineMode:
NO CCR marker / retrieval round-trip, so the output is safe to forward
straight to a provider (Kong-sidecar use case)."""
- def _big_tool_message(self):
+ def _big_tool_message(self, repetitions=1):
large_data = json.dumps(
[
{
@@ -448,8 +448,9 @@ class TestCompressEndpointLossyInlineMode:
f"category=electronics, price={i * 10.99:.2f}, stock={i * 5}.",
"tags": ["electronics", "sale", "featured", "new-arrival"],
}
- for i in range(200)
+ for i in range(200 // repetitions)
]
+ * repetitions
)
return [
{"role": "user", "content": "What items are available?"},
@@ -459,9 +460,7 @@ class TestCompressEndpointLossyInlineMode:
@pytest.fixture
def client(self):
- # disable_kompress keeps the real ONNX model out of the test: marker
- # suppression is exercised by SmartCrusher (pure-Python) regardless, and
- # the mode inherits enable_kompress from config so this stays fast.
+ # Exercise the actual native JSON steps without unrelated ML inference.
config = ProxyConfig(
optimize=True,
cache_enabled=False,
@@ -473,8 +472,11 @@ class TestCompressEndpointLossyInlineMode:
with TestClient(app, base_url="http://127.0.0.1", client=("127.0.0.1", 12345)) as c:
yield c
- def test_lossy_inline_emits_no_ccr_markers(self, client):
- messages = self._big_tool_message()
+ @pytest.mark.parametrize(
+ ("repetitions", "strategy"), [(1, "json_minifier"), (20, "smart_crusher")]
+ )
+ def test_lossy_inline_emits_no_ccr_markers(self, client, repetitions, strategy):
+ messages = self._big_tool_message(repetitions=repetitions)
response = client.post(
"/v1/compress",
json={"messages": messages, "model": "gpt-4", "config": {"mode": "lossy_inline"}},
@@ -497,6 +499,37 @@ class TestCompressEndpointLossyInlineMode:
assert data["tokens_after"] < data["tokens_before"]
assert data["tokens_saved"] == data["tokens_before"] - data["tokens_after"]
+ original_rows = json.loads(messages[1]["content"])
+ tool = next(message for message in data["messages"] if message["role"] == "tool")
+ compressed_rows, _ = json.JSONDecoder().raw_decode(tool["content"])
+ if repetitions == 1:
+ assert compressed_rows == original_rows
+ else:
+ assert len(compressed_rows) < len(original_rows)
+ assert {json.dumps(row, sort_keys=True) for row in compressed_rows} == {
+ json.dumps(row, sort_keys=True) for row in original_rows
+ }
+ assert strategy in {transform.split(":")[1] for transform in data["transforms_applied"]}
+
+ def test_lossy_inline_preserves_no_op_tool_content(self, client):
+ messages = self._big_tool_message()
+ messages[1]["content"] = json.dumps(
+ json.loads(messages[1]["content"]), separators=(",", ":")
+ )
+ original = messages[1]["content"]
+ response = client.post(
+ "/v1/compress",
+ json={"messages": messages, "model": "gpt-4", "config": {"mode": "lossy_inline"}},
+ )
+ assert response.status_code == 200
+ data = response.json()
+
+ tool = next(message for message in data["messages"] if message["role"] == "tool")
+ assert tool["content"] == original
+ assert data["tokens_saved"] == 0
+ assert data["tokens_after"] == data["tokens_before"]
+ assert data["ccr_hashes"] == []
+
def test_lossless_then_lossy_alias(self, client):
"""The spelled-out alias selects the same mode."""
messages = self._big_tool_message()
diff --git a/tests/test_recursive_json.py b/tests/test_recursive_json.py
index 53a8ff8cb..01275ada3 100644
--- a/tests/test_recursive_json.py
+++ b/tests/test_recursive_json.py
@@ -1,6 +1,4 @@
-"""Unit tests for headroom.transforms.recursive_json — the structural (embedded)
-JSON routing step. Uses a fake dispatch so the mechanism is tested in isolation
-from the real compressors."""
+"""Embedded JSON routing: preserved surroundings and per-adopted-span recovery."""
from __future__ import annotations
@@ -13,62 +11,89 @@ from headroom.transforms.recursive_json import (
)
-def _upper_dispatch(span: str) -> str | None:
- """Fake compressor: returns a shorter deterministic stand-in for any span."""
- try:
- v = json.loads(span)
- except ValueError:
- return None
- return f"<TABLE n={len(v)}>" if isinstance(v, list) else "<OBJ>"
+def _minify_dispatch(span: str) -> tuple[str, bool]:
+ return json.dumps(json.loads(span), separators=(",", ":")), True
def test_embedded_json_routed_and_surroundings_exact() -> None:
- payload = json.dumps([{"id": i, "ok": True} for i in range(6)], separators=(",", ":"))
+ rows = [{"id": i, "ok": True} for i in range(6)]
+ payload = json.dumps(rows, indent=2)
content = f"Fetched rows from API:\n{payload}\nDone (200 OK)."
- out = route_embedded_json(content, _upper_dispatch)
- assert out is not None
- assert out.startswith("Fetched rows from API:\n")
- assert out.endswith("\nDone (200 OK).")
- assert "<TABLE n=6>" in out
+ assert route_embedded_json(content, _minify_dispatch) == (
+ f"Fetched rows from API:\n{json.dumps(rows, separators=(',', ':'))}\nDone (200 OK).",
+ True,
+ )
+
+
+def test_noop_json_span_does_not_block_later_compaction() -> None:
+ compact = json.dumps([{"id": i, "ok": True} for i in range(6)], separators=(",", ":"))
+ rows = [{"id": i, "message": f"entry-{i}"} for i in range(20, 26)]
+ padded = json.dumps(rows, indent=2)
+ leading, between, trailing = "Début 🚀\r\nfirst: ", "\r\nsecond: ", "\r\nFin."
+ content = leading + compact + between + padded + trailing
+
+ assert route_embedded_json(content, _minify_dispatch) == (
+ leading + compact + between + json.dumps(rows, separators=(",", ":")) + trailing,
+ True,
+ )
def test_ccr_marker_span_passed_through() -> None:
- # A span already carrying a CCR marker must never be re-routed (R1).
content = 'prefix [{"a":1,"b":2},{"a":3,"b":"<<ccr:deadbeef,json,900>>"}] suffix'
- out = route_embedded_json(content, _upper_dispatch)
- assert out is None # only span contains a marker → skipped → nothing to do
+ assert route_embedded_json(content, _minify_dispatch) is None
def test_no_json_is_noop() -> None:
- assert route_embedded_json("just prose, nothing structured here", _upper_dispatch) is None
+ assert route_embedded_json("just prose, nothing structured here", _minify_dispatch) is None
def test_whole_block_json_is_callers_job() -> None:
- # A block that IS a single JSON value is skipped (routed by the caller).
content = json.dumps([{"a": i} for i in range(5)], separators=(",", ":"))
- assert route_embedded_json(content, _upper_dispatch) is None
+ assert route_embedded_json(content, _minify_dispatch) is None
def test_benefit_gate_declines_when_not_smaller() -> None:
- payload = json.dumps([{"a": i} for i in range(5)], separators=(",", ":"))
- content = f"x {payload} y"
- # Dispatch that returns something LARGER → must be declined (outcome gate).
- assert route_embedded_json(content, lambda s: s + " " * 999) is None
+ payload = json.dumps([{"a": i} for i in range(5)])
+ assert route_embedded_json(f"x {payload} y", lambda s: (s + " " * 999, True)) is None
+
+
+def test_unmarked_loss_cannot_borrow_another_spans_marker() -> None:
+ first = json.dumps([{"id": i} for i in range(10)], indent=2)
+ second = json.dumps([{"value": i} for i in range(10)], indent=2)
+
+ def dispatch(span: str) -> tuple[str, bool]:
+ rows = json.loads(span)
+ if "id" in rows[0]:
+ return "<<ccr:deadbeef,json,100>>", True
+ return json.dumps(rows[:1]), False
+
+ for left, right in ((first, second), (second, first)):
+ result = route_embedded_json(f"first: {left}\nsecond: {right}", dispatch, tok=len)
+ assert result is not None
+ text, reversible = result
+ assert "<<ccr:deadbeef,json,100>>" in text
+ assert '{"value": 0}' in text and '{"value": 9}' not in text
+ assert reversible is False
+
+
+def test_rejected_lossy_candidate_does_not_taint_adopted_minification() -> None:
+ first = json.dumps([{"id": i} for i in range(10)], indent=2)
+ second = json.dumps([{"value": i} for i in range(10)], indent=2)
+ def dispatch(span: str) -> tuple[str, bool]:
+ rows = json.loads(span)
+ if "id" in rows[0]:
+ return json.dumps(rows[:1]) + " " * len(span), False
+ return _minify_dispatch(span)
-def test_deterministic() -> None:
- payload = json.dumps([{"k": i} for i in range(8)], separators=(",", ":"))
- content = f"a {payload} b {payload} c"
- r1 = route_embedded_json(content, _upper_dispatch)
- r2 = route_embedded_json(content, _upper_dispatch)
- assert r1 == r2 and r1 is not None
- assert r1.count("<TABLE n=8>") == 2 # both embedded spans routed
+ assert route_embedded_json(f"first: {first}\nsecond: {second}", dispatch, tok=len) == (
+ f"first: {first}\nsecond: {json.dumps(json.loads(second), separators=(',', ':'))}",
+ True,
+ )
def test_scalar_array_not_routed() -> None:
- # array of scalars is not a "routable" JSON shape (no dict rows)
- content = "nums: [1,2,3,4,5,6,7,8] done"
- assert route_embedded_json(content, _upper_dispatch) is None
+ assert route_embedded_json("nums: [1,2,3,4,5,6,7,8] done", _minify_dispatch) is None
def test_json_document_spans_finds_containers_anywhere() -> None:
diff --git a/tests/test_relevance_split.py b/tests/test_relevance_split.py
index 6f7e5ef5b..3e2e57132 100644
--- a/tests/test_relevance_split.py
+++ b/tests/test_relevance_split.py
@@ -215,7 +215,9 @@ def test_router_lossless_mode_folds_only_no_drop():
# irrelevant "widget" records must be PRESERVED, not silently dropped, and the
# Kompress tail stub must never run.
r = _router(split_on=True) # lossless mode
- out, _, chain = r._apply_strategy_to_content(_SEARCH, CompressionStrategy.SEARCH, "oauth token")
+ out, _, chain, _reversible = r._apply_strategy_to_content(
+ _SEARCH, CompressionStrategy.SEARCH, "oauth token"
+ )
assert chain == ["lossless_search"]
assert "oauth token" in out # relevant records kept
assert "widget" in out # irrelevant tail ALSO kept — no silent drop in lossless mode
@@ -228,7 +230,9 @@ def test_router_relevance_split_fires_in_ccr_mode():
# #1721); the _try_ml_compressor stub stands in for it here. Proves the
# split is mode-agnostic, not lossless-only.
r = _router(split_on=True, lossless=False)
- out, _, chain = r._apply_strategy_to_content(_SEARCH, CompressionStrategy.SEARCH, "oauth token")
+ out, _, chain, _reversible = r._apply_strategy_to_content(
+ _SEARCH, CompressionStrategy.SEARCH, "oauth token"
+ )
assert chain == ["search", "relevance_split"]
assert "oauth token" in out
assert "[TAIL]" in out
@@ -237,14 +241,18 @@ def test_router_relevance_split_fires_in_ccr_mode():
def test_router_diff_stays_pure_lossless():
r = _router(split_on=True)
diff = "diff --git a/x b/x\nindex 111..222 100644\n@@ -1 +1 @@\n-old widget\n+new oauth token\n"
- _, _, chain = r._apply_strategy_to_content(diff, CompressionStrategy.DIFF, "oauth token")
+ _, _, chain, _reversible = r._apply_strategy_to_content(
+ diff, CompressionStrategy.DIFF, "oauth token"
+ )
assert "relevance_split" not in chain # Kompressing hunks would break apply
assert chain == ["lossless_diff"]
def test_router_split_can_be_disabled():
r = _router(split_on=False)
- _, _, chain = r._apply_strategy_to_content(_SEARCH, CompressionStrategy.SEARCH, "oauth token")
+ _, _, chain, _reversible = r._apply_strategy_to_content(
+ _SEARCH, CompressionStrategy.SEARCH, "oauth token"
+ )
assert "relevance_split" not in chain
diff --git a/tests/test_router_external_dispatch.py b/tests/test_router_external_dispatch.py
index 47b2a66da..27ee1ed15 100644
--- a/tests/test_router_external_dispatch.py
+++ b/tests/test_router_external_dispatch.py
@@ -33,7 +33,6 @@ from headroom.transforms.compressor_registry import (
CompressOutput,
)
from headroom.transforms.content_router import (
- CompressionStrategy,
ContentRouter,
ContentRouterConfig,
)
@@ -250,21 +249,6 @@ def test_native_misclassification_keeps_git_status_format(_memory_ccr, monkeypat
assert detected.metadata["format"] == "git_status"
-def test_external_dispatch_via_apply_strategy_returns_normal_shape(_memory_ccr):
- comp = _RecordingExternal(name="ext_json", content_types=("application/json",))
- router = _router_with_external(comp, ["ext_json"])
-
- compressed, tokens, chain = router._apply_strategy_to_content(
- _JSON_ARRAY, CompressionStrategy.SMART_CRUSHER, ""
- )
-
- assert chain == ["external:ext_json"]
- assert "external-compressed" in compressed
- # Tokens counted with the router's OWN estimator (a positive int), not the
- # compressor's self-report.
- assert isinstance(tokens, int) and tokens > 0
-
-
# ─────────────────────────── (b) recoverable map ─────────────────────────────
diff --git a/tests/test_router_registry_dispatch.py b/tests/test_router_registry_dispatch.py
index 92257a105..097409a64 100644
--- a/tests/test_router_registry_dispatch.py
+++ b/tests/test_router_registry_dispatch.py
@@ -1,360 +1,25 @@
-"""Byte-identical differential tests for registry-resolved built-in dispatch.
+"""Unavailable HTML extraction must preserve tool output."""
-The content router now dispatches the SIMPLE built-in strategies (CONFIG, LOG,
-SEARCH, TABULAR) through the compressor registry instead of a hardcoded direct
-``self._get_*().compress(...)`` call in ``_apply_strategy_to_content``. Each
-built-in adapter delegates to the SAME ``_get_*`` getter+method with the SAME
-arguments, so registry-resolved dispatch must be byte-identical to the historical
-direct dispatch: same compressed content, same token count, same single-entry
-``strategy_chain``.
-
-Each FLIPPED strategy has a differential test comparing the router's dispatch
-output to the built-in's direct output obtained via its ``_get_*`` getter — i.e.
-"registry dispatch == old dispatch". SMART_CRUSHER is now also flipped (its
-``.crush`` primary invocation goes through the registry while the shared
-Kompress→Log fallback block stays direct); KOMPRESS/TEXT now dispatch via the
-registry too — the ``kompress`` adapter delegates to the SAME ``_try_ml_compressor``
-call with ``question`` forwarded via ``config['question']``, so CONTENT is
-preserved and only the reported token metric changed. See
-``test_router_registry_smartcrusher.py`` for the full SMART_CRUSHER fallback-chain
-and KOMPRESS/TEXT differential coverage.
-
-Offline guardrails:
- * No real ML/ONNX/HF inference — the KOMPRESS ML boundary is mocked.
- * The flipped strategies shrink their representative content, so no zero-savings
- Kompress fallback fires (that would touch the ML boundary and append KOMPRESS
- to the chain).
- * ``lossless_then_lossy`` and ``relevance_split`` are off so the if/elif branch
- is the terminal path; STAGE 0 (``_lossless_first``) is neutralized so search/
- log folds don't return before the branch. Both are shared, unchanged code.
- * The broad ``content_router``/``compression`` -k selection is NOT exercised
- (it hangs on HF-Hub/ONNX).
-"""
-
-from __future__ import annotations
-
-import json
from types import SimpleNamespace
import pytest
from headroom.transforms.content_router import (
- _BUILTIN_COMPRESSOR_DESCRIPTORS,
CompressionStrategy,
ContentRouter,
ContentRouterConfig,
- _estimate_tokens,
)
-def _router() -> ContentRouter:
- """Router whose if/elif branch is the terminal dispatch path.
-
- ``relevance_split`` off (no LOG/SEARCH relevance split) and
- ``lossless_then_lossy`` off (no lossy layer on top of a strategy result) so a
- successful strategy result returns directly. ``ccr_inject_marker`` off makes
- the compressed output deterministic and marker-free; it is applied identically
- to the direct reference and the dispatch router, so the differential holds
- regardless of its value.
- """
- return ContentRouter(
- ContentRouterConfig(
- relevance_split=False,
- lossless_then_lossy=False,
- ccr_inject_marker=False,
- )
- )
-
-
-def _isolate_branch(monkeypatch: pytest.MonkeyPatch, router: ContentRouter) -> None:
- """Neutralize STAGE 0 so the if/elif branch under test is exercised.
-
- ``_lossless_first`` runs unconditionally and can fold search/log content,
- returning before the if/elif. It is shared, unchanged code (the flip only
- touches the branch bodies), so forcing it to a no-op isolates what the flip
- actually changed without altering the branch semantics.
- """
- monkeypatch.setattr(router, "_lossless_first", lambda content, strategy: (content, None))
-
-
-# Representative content per type. The flipped strategies must SHRINK this so the
-# fallback-eligible strategies (TABULAR/CONFIG) don't trip the zero-savings
-# Kompress fallback (which would append KOMPRESS to the chain).
-_SEARCH = "\n".join(f"src/file{i}.py:{i}: def func{i}(): return {i}" for i in range(30))
-_LOG = (
- "\n".join(f"2024-01-01 12:00:{i:02d} INFO task {i}" for i in range(30))
- + "\n"
- + "\n".join("identical repeated line" for _ in range(25))
-)
-# A markdown table the tabular compressor actually shrinks (schema-fold), and a
-# repetitive INI the config compressor actually shrinks (block-fold) — so these
-# fallback-eligible strategies produce a real token saving and the shared
-# zero-savings Kompress fallback does NOT fire (chain stays single-entry).
-_TABULAR = "| id | name | status | score |\n|----|------|--------|-------|\n" + "\n".join(
- f"| {i} | row{i} | ok | {i * 3} |" for i in range(60)
-)
-_CONFIG = "\n".join(
- f"[section_{i}]\nname = svc{i}\ntimeout = 30\nretries = 3\nverbose = false\nregion = us-east-1"
- for i in range(40)
-)
-
-
-# ───────────────────────── flipped (registry dispatch) ────────────────────────
-
-
-def test_search_router_dispatch_matches_direct(monkeypatch: pytest.MonkeyPatch) -> None:
- router = _router()
- _isolate_branch(monkeypatch, router)
- context, bias = "func", 1.0
- # OLD dispatch reference: same getter + method the branch used before the flip.
- direct = (
- _router()._get_search_compressor().compress(_SEARCH, context=context, bias=bias).compressed
- )
- out, tokens, chain = router._apply_strategy_to_content(
- _SEARCH, CompressionStrategy.SEARCH, context, bias=bias
- )
- assert out == direct
- assert tokens == _estimate_tokens(direct)
- assert chain == [CompressionStrategy.SEARCH.value]
-
-
-def test_log_router_dispatch_matches_direct(monkeypatch: pytest.MonkeyPatch) -> None:
- router = _router()
- _isolate_branch(monkeypatch, router)
- bias = 1.0
- direct = _router()._get_log_compressor().compress(_LOG, bias=bias).compressed
- out, tokens, chain = router._apply_strategy_to_content(
- _LOG, CompressionStrategy.LOG, "", bias=bias
- )
- assert out == direct
- assert tokens == _estimate_tokens(direct)
- assert chain == [CompressionStrategy.LOG.value]
-
-
-def test_tabular_router_dispatch_matches_direct(monkeypatch: pytest.MonkeyPatch) -> None:
- router = _router()
- _isolate_branch(monkeypatch, router)
- context, bias = "q", 1.0
- direct = (
- _router()
- ._get_tabular_compressor()
- .compress(_TABULAR, context=context, bias=bias)
- .compressed
- )
- out, tokens, chain = router._apply_strategy_to_content(
- _TABULAR, CompressionStrategy.TABULAR, context, bias=bias
- )
- assert out == direct
- assert tokens == _estimate_tokens(direct)
- # Fallback-eligible, but a real shrink means no zero-savings Kompress fallback.
- assert chain == [CompressionStrategy.TABULAR.value]
- assert len(out) < len(_TABULAR)
-
-
-def test_config_router_dispatch_matches_direct(monkeypatch: pytest.MonkeyPatch) -> None:
- router = _router()
- _isolate_branch(monkeypatch, router)
- context, bias = "q", 1.0
- direct = (
- _router()._get_config_compressor().compress(_CONFIG, context=context, bias=bias).compressed
- )
- out, tokens, chain = router._apply_strategy_to_content(
- _CONFIG, CompressionStrategy.CONFIG, context, bias=bias
- )
- assert out == direct
- # CONFIG's historical metric was len(text.split()) — a WORD count, which this
- # assertion was written to preserve across the registry-dispatch flip. That
- # was right for the flip and wrong as a metric: the value is divided by
- # `original_tokens`, which comes from _estimate_tokens, so a word numerator
- # over a token denominator made a byte-identical no-op score ~0.36 and get
- # accepted as a large saving. Now measured in the denominator's unit.
- assert tokens == _estimate_tokens(direct)
- assert chain == [CompressionStrategy.CONFIG.value]
- assert len(out) < len(_CONFIG)
-
-
-# ─────────────────────────── deferred (unchanged) ────────────────────────────
-
-
-def test_smart_crusher_router_dispatch_matches_direct(monkeypatch: pytest.MonkeyPatch) -> None:
- # SMART_CRUSHER is now FLIPPED: its primary ``.crush`` invocation routes
- # through the registry "smart_crusher" adapter (which delegates to the SAME
- # getter + ``.crush(query=..., bias=...)``), while the shared Kompress→Log
- # fallback block stays direct. Registry dispatch must be byte-identical to the
- # historical direct crush. A JSON array shrinks, so no fallback fires and the
- # chain stays single.
- router = _router()
- _isolate_branch(monkeypatch, router)
- content = json.dumps(
- [{"id": i, "status": "ok", "level": "INFO", "value": i * 2} for i in range(40)]
- )
- direct = _router()._get_smart_crusher().crush(content, query="q", bias=1.0).compressed
- out, _tokens, chain = router._apply_strategy_to_content(
- content, CompressionStrategy.SMART_CRUSHER, "q", bias=1.0
- )
- assert out == direct
- assert chain == [CompressionStrategy.SMART_CRUSHER.value]
-
-
-def _fallback_router() -> ContentRouter:
- """Router with CODE_AWARE routing enabled and the if/elif branch terminal.
-
- Same isolation as :func:`_router` (no relevance split, no lossy layer, markers
- off) but with ``enable_code_aware=True`` since that flag gates CODE_AWARE
- ROUTING (default off) rather than the getter. HTML routing is on by default.
- """
- return ContentRouter(
- ContentRouterConfig(
- relevance_split=False,
- lossless_then_lossy=False,
- ccr_inject_marker=False,
- enable_code_aware=True,
- )
- )
-
-
-# ─────────────── flipped fallback strategies: CODE_AWARE / HTML ───────────────
-
-
-def test_code_aware_router_dispatch_matches_direct(monkeypatch: pytest.MonkeyPatch) -> None:
- # CODE_AWARE-SUCCEEDS: the flip routes through the registry "code_aware"
- # adapter, which delegates to the SAME getter+method with language via config.
- # A deterministic fake getter keeps this offline (no tree-sitter) and records
- # the exact call args, proving content/language/context flow through unchanged.
- router = _fallback_router()
- _isolate_branch(monkeypatch, router)
- seen: dict[str, object] = {}
- shrunk = "def foo(): ... # compressed body"
-
- def _fake_compress(content: str, language: object = None, context: str = "") -> SimpleNamespace:
- seen.update(content=content, language=language, context=context)
- return SimpleNamespace(compressed=shrunk)
-
- monkeypatch.setattr(
- router, "_get_code_compressor", lambda: SimpleNamespace(compress=_fake_compress)
- )
- # Sentinel ML: if the flip WRONGLY dropped into a Kompress fallback we'd see
- # KOMPRESS appended to the chain; the chain assertion catches it.
- monkeypatch.setattr(
- router, "_try_ml_compressor", lambda *a, **k: ("KOMPRESS_SENTINEL", 999_999)
- )
-
- content = "def foo():\n " + "x = 1\n " * 40 + "return x\n"
- out, tokens, chain = router._apply_strategy_to_content(
- content, CompressionStrategy.CODE_AWARE, "q", language="python", bias=1.0
- )
- assert out == shrunk
- # Measured in the router's token estimate, the unit of original_tokens.
- assert tokens == _estimate_tokens(shrunk)
- assert chain == [CompressionStrategy.CODE_AWARE.value]
- # The flip forwarded content, language, and context through the registry adapter.
- assert seen == {"content": content, "language": "python", "context": "q"}
-
-
-def test_code_aware_unavailable_falls_back_to_kompress(monkeypatch: pytest.MonkeyPatch) -> None:
- # CODE_AWARE-RETURNS-NONE: with the code compressor UNAVAILABLE (tree-sitter
- # missing) the branch's local `compressed` stays None, so the EXISTING inline
- # Kompress fallback runs — the flip touches none of that logic. Mock
- # _try_ml_compressor so NO real ML runs, and assert the SAME strategy_chain
- # ([code_aware, kompress]) the historical direct dispatch produced.
- router = _fallback_router()
- _isolate_branch(monkeypatch, router)
- monkeypatch.setattr(router, "_get_code_compressor", lambda: None)
- monkeypatch.setattr(
- router,
- "_try_ml_compressor",
- lambda content, ctx, q: ("KOMPRESSED::" + content, 3),
- )
- content = "def foo():\n return 1\n"
- out, tokens, chain = router._apply_strategy_to_content(
- content, CompressionStrategy.CODE_AWARE, "ctx", language=None, bias=1.0
- )
- assert out == "KOMPRESSED::" + content
- assert tokens == 3
- assert chain == [CompressionStrategy.CODE_AWARE.value, CompressionStrategy.KOMPRESS.value]
-
-
-def test_html_router_dispatch_matches_direct(monkeypatch: pytest.MonkeyPatch) -> None:
- # HTML-EXTRACT-SUCCEEDS: the flip routes through the registry "html" adapter,
- # which delegates to the SAME getter + extract(). A deterministic fake getter
- # keeps this offline (no trafilatura) and records the call arg.
- router = _fallback_router()
- _isolate_branch(monkeypatch, router)
- extracted = "Extracted article body text that trafilatura would return."
- seen: dict[str, object] = {}
-
- def _fake_extract(content: str) -> SimpleNamespace:
- seen["content"] = content
- return SimpleNamespace(extracted=extracted)
-
- monkeypatch.setattr(
- router, "_get_html_extractor", lambda: SimpleNamespace(extract=_fake_extract)
- )
- content = "<html><body><article><p>hello world</p></article></body></html>"
- out, tokens, chain = router._apply_strategy_to_content(
- content, CompressionStrategy.HTML, "", bias=1.0
- )
- assert out == extracted
- assert tokens == _estimate_tokens(extracted)
- assert chain == [CompressionStrategy.HTML.value]
- assert seen == {"content": content}
-
-
def test_html_extract_none_falls_through_to_passthrough(monkeypatch: pytest.MonkeyPatch) -> None:
- # HTML-EXTRACT-NONE: when extraction yields None the adapter reports
- # compressed=False, the branch's local `compressed` collapses to None, and the
- # function falls through to the bottom passthrough exactly as the historical
- # `result.extracted is None` path — chain [html, passthrough], content verbatim.
- router = _fallback_router()
- _isolate_branch(monkeypatch, router)
+ router = ContentRouter(ContentRouterConfig(relevance_split=False, lossless_then_lossy=False))
monkeypatch.setattr(
router,
"_get_html_extractor",
lambda: SimpleNamespace(extract=lambda content: SimpleNamespace(extracted=None)),
)
- # Sentinel ML so we'd notice if the None path wrongly reached a lossy compressor.
- monkeypatch.setattr(router, "_try_ml_compressor", lambda *a, **k: ("KOMPRESS_SENTINEL", 1))
content = "<html><body><script>no extractable article body</script></body></html>"
- out, tokens, chain = router._apply_strategy_to_content(
- content, CompressionStrategy.HTML, "", bias=1.0
+ out, _tokens, _chain, _reversible = router._apply_strategy_to_content(
+ content, CompressionStrategy.HTML, ""
)
assert out == content
- assert tokens == _estimate_tokens(content)
- assert chain == [CompressionStrategy.HTML.value, CompressionStrategy.PASSTHROUGH.value]
-
-
-def test_diff_deferred_no_registry_entry() -> None:
- # DIFF is DEFERRED: there is no "diff" built-in adapter/descriptor in the
- # registry inventory, so registry resolution would return None and fall back
- # to raw content — NOT byte-identical. It stays on its direct dispatch until a
- # diff built-in adapter lands (PR-A scope).
- router = _router()
- assert router.compressor_registry.get("diff") is None
- assert "diff" not in {d.name for d in _BUILTIN_COMPRESSOR_DESCRIPTORS}
-
-
-def test_kompress_registry_dispatch_matches_direct(monkeypatch: pytest.MonkeyPatch) -> None:
- # KOMPRESS now dispatches via the registry "kompress" adapter, which still
- # delegates to _try_ml_compressor (the ML boundary). Mock the underlying model
- # so no real ONNX/HF inference runs, and assert the compressed CONTENT is
- # byte-identical to the direct _try_ml_compressor call (question is None here).
- router = _router()
- _isolate_branch(monkeypatch, router)
- fake = SimpleNamespace(
- is_ready=lambda: True,
- ensure_background_load=lambda: None,
- compress=lambda text, **kwargs: SimpleNamespace(
- compressed="KOMPRESSED::" + text, compressed_tokens=7
- ),
- )
- monkeypatch.setattr(router, "_get_kompress", lambda: fake)
- content = "some plain text that the ML model would compress. " * 4
- out, _tokens, chain = router._apply_strategy_to_content(
- content, CompressionStrategy.KOMPRESS, "", bias=1.0
- )
- assert out == "KOMPRESSED::" + content
- assert chain == [CompressionStrategy.KOMPRESS.value]
- # CONTENT is byte-identical to the direct ML call (registry round-trip preserves
- # content); the approved token-metric change is covered in
- # test_router_registry_smartcrusher.py.
- assert out == router._try_ml_compressor(content, "", None)[0]
diff --git a/tests/test_router_registry_smartcrusher.py b/tests/test_router_registry_smartcrusher.py
index d80684a31..8705bf516 100644
--- a/tests/test_router_registry_smartcrusher.py
+++ b/tests/test_router_registry_smartcrusher.py
@@ -1,43 +1,8 @@
-"""Differential tests for the SMART_CRUSHER / KOMPRESS / TEXT registry flip.
-
-SMART_CRUSHER flips the PRIMARY compressor invocation to the compressor registry
-(``_registry_compress("smart_crusher", ...)``), while the shared post-strategy
-Kompress -> Log fallback block stays a direct dispatch. KOMPRESS and TEXT now ALSO
-dispatch via the registry (``_registry_compress("kompress", ...)``): the
-``kompress`` built-in adapter delegates to the SAME ``_try_ml_compressor(content,
-context, question)`` the router historically called, with ``question`` forwarded
-via ``CompressInput.config['question']`` (previously the adapter dropped it,
-passing ``None`` — that latent bug is fixed here). Content is therefore PRESERVED
-byte-for-byte against the direct ``_try_ml_compressor`` call.
-
-The ONE approved, non-byte-identical change is the token metric: KOMPRESS/TEXT now
-report ``_estimate_tokens(output.content)`` (the router's calibrated estimate)
-instead of ``_try_ml_compressor``'s tuple token count (Kompress's own
-``compressed_tokens``). No content/routing/fallback/lossless-then-lossy decision
-reads that metric for these two branches, so only the reported number changes.
-
-Every path asserts registry-dispatch CONTENT == the historical direct-dispatch
-content (and, via recorded call args, that query/bias/question flow through), and
-that the token metric equals ``_estimate_tokens(output.content)``.
-
-Offline guardrails:
- * No real ML/ONNX/HF inference — the ML boundary is mocked at
- ``_try_ml_compressor`` (SMART_CRUSHER fallback tests) or ``_get_kompress``
- (KOMPRESS/TEXT tests, which exercise the real ``_try_ml_compressor`` with a
- fake in-memory model).
- * SmartCrusher itself is pure-Python (no ML), so the SMART_CRUSHER success path
- runs the real crusher on a small JSON array.
- * ``relevance_split`` / ``lossless_then_lossy`` off and STAGE 0
- (``_lossless_first``) neutralized so the if/elif branch under test is the
- terminal path. All of that is shared, unchanged code.
- * The broad ``content_router``/``compression`` -k selection is NOT exercised
- (it hangs on HF-Hub/ONNX).
-"""
+"""Lossless JSON precedence and real no-saving fallback boundaries."""
from __future__ import annotations
import json
-from types import SimpleNamespace
import pytest
@@ -49,255 +14,35 @@ from headroom.transforms.content_router import (
)
-def _router() -> ContentRouter:
- """Router whose if/elif branch is the terminal dispatch path.
-
- ``relevance_split`` off, ``lossless_then_lossy`` off, and
- ``ccr_inject_marker`` off make the compressed output deterministic and
- marker-free; the same config is applied to the direct reference and the
- dispatch router, so the differential holds regardless.
- """
- return ContentRouter(
[email protected]("pretty", [True, False], ids=["minification", "compact-fallback"])
+def test_json_minification_precedes_fallback_without_losing_records(pretty: bool) -> None:
+ records = [
+ {
+ "id": i,
+ "status": "ok",
+ "level": "ERROR" if i == 20 else "INFO",
+ "message": f"CRITICAL_{i} transaction completed",
+ }
+ for i in range(60)
+ ]
+ content = (
+ json.dumps(records, indent=2) if pretty else json.dumps(records, separators=(",", ":"))
+ )
+ router = ContentRouter(
ContentRouterConfig(
- relevance_split=False,
- lossless_then_lossy=False,
+ smart_crusher_with_compaction=False,
+ smart_crusher_max_items_after_crush=1000,
+ enable_kompress=False,
ccr_inject_marker=False,
)
)
-
-
-def _isolate_branch(monkeypatch: pytest.MonkeyPatch, router: ContentRouter) -> None:
- """Neutralize STAGE 0 (``_lossless_first``) so the if/elif branch is exercised.
-
- ``_lossless_first`` is shared, unchanged code (the flip only touches the branch
- body), so forcing it to a no-op isolates what the flip actually changed.
- """
- monkeypatch.setattr(router, "_lossless_first", lambda content, strategy: (content, None))
-
-
-# A JSON array the real SmartCrusher shrinks (so the fallback chain stays single).
-_JSON = json.dumps([{"id": i, "status": "ok", "level": "INFO", "value": i * 2} for i in range(40)])
-
-
-# ─────────────────────── SMART_CRUSHER: flipped (registry) ────────────────────
-
-
-def test_smart_crusher_success_matches_direct(monkeypatch: pytest.MonkeyPatch) -> None:
- # SMART_CRUSHER-SUCCEEDS: the flip routes the primary ``.crush`` through the
- # registry "smart_crusher" adapter, which delegates to the SAME getter +
- # ``.crush(query=..., bias=...)``. Differential: registry dispatch == a direct
- # crush on an independent router. A real shrink means no fallback fires.
- router = _router()
- _isolate_branch(monkeypatch, router)
- context, bias = "q", 1.0
- direct = _router()._get_smart_crusher().crush(_JSON, query=context, bias=bias).compressed
- out, tokens, chain = router._apply_strategy_to_content(
- _JSON, CompressionStrategy.SMART_CRUSHER, context, bias=bias
- )
- assert out == direct
- assert tokens == _estimate_tokens(direct)
- assert chain == [CompressionStrategy.SMART_CRUSHER.value]
- assert len(out) < len(_JSON)
-
-
-def test_smart_crusher_forwards_query_and_bias(monkeypatch: pytest.MonkeyPatch) -> None:
- # The flip must forward content, query (==context), and bias to ``.crush``
- # unchanged. A fake crusher records the exact call args, proving the registry
- # adapter's ``query=inp.query`` / ``bias=budget['bias']`` reproduce the direct
- # ``crush(content, query=context, bias=bias)`` call.
- router = _router()
- _isolate_branch(monkeypatch, router)
- seen: dict[str, object] = {}
-
- def _crush(content: str, query: str = "", bias: float = 1.0) -> SimpleNamespace:
- seen.update(content=content, query=query, bias=bias)
- return SimpleNamespace(compressed="CRUSHED " + content[:5])
-
- monkeypatch.setattr(router, "_get_smart_crusher", lambda: SimpleNamespace(crush=_crush))
- # Sentinel ML: if the flip WRONGLY dropped into the Kompress fallback we'd see
- # KOMPRESS appended to the chain; the chain assertion catches it.
- monkeypatch.setattr(
- router, "_try_ml_compressor", lambda *a, **k: ("KOMPRESS_SENTINEL", 999_999)
- )
-
- content = "x" * 200
- out, tokens, chain = router._apply_strategy_to_content(
- content, CompressionStrategy.SMART_CRUSHER, "myquery", bias=0.7
- )
- assert out == "CRUSHED " + content[:5]
- assert tokens == _estimate_tokens("CRUSHED " + content[:5])
- assert chain == [CompressionStrategy.SMART_CRUSHER.value]
- assert seen == {"content": content, "query": "myquery", "bias": 0.7}
-
-
-def test_smart_crusher_kompress_fallback_matches_direct(monkeypatch: pytest.MonkeyPatch) -> None:
- # SMART_CRUSHER-NO-SHRINK -> KOMPRESS: the crusher returns the content
- # unchanged (no net saving), so the SHARED, UNCHANGED post-strategy block runs
- # the Kompress fallback. The flip only sets ``compressed``/``compressed_tokens``
- # on entry — identically to the direct crush — so the fallback fires exactly as
- # before: chain [smart_crusher, kompress], Kompress output adopted.
- router = _router()
- _isolate_branch(monkeypatch, router)
- monkeypatch.setattr(
- router,
- "_get_smart_crusher",
- lambda: SimpleNamespace(crush=lambda c, query="", bias=1.0: SimpleNamespace(compressed=c)),
+ compressed, tokens, chain, _reversible = router._apply_strategy_to_content(
+ content, CompressionStrategy.SMART_CRUSHER, "CRITICAL_20"
)
- # Kompress shrinks (fallback_tokens < compressed_tokens) so it is adopted.
- # The fallback is measured with the router's estimator, not the count the
- # ML compressor reports, so the mock must actually return less text.
- kompressed = "KOMPRESSED::" + _JSON[:40]
- monkeypatch.setattr(router, "_try_ml_compressor", lambda c, ctx, q: (kompressed, 3))
- out, tokens, chain = router._apply_strategy_to_content(
- _JSON, CompressionStrategy.SMART_CRUSHER, "ctx", bias=1.0
- )
- assert out == kompressed
- assert tokens == _estimate_tokens(kompressed)
- assert chain == [
- CompressionStrategy.SMART_CRUSHER.value,
- CompressionStrategy.KOMPRESS.value,
- ]
-
-def test_smart_crusher_log_fallback_matches_direct(monkeypatch: pytest.MonkeyPatch) -> None:
- # SMART_CRUSHER-NO-SHRINK -> KOMPRESS-NO-SHRINK -> LOG: crusher passes content
- # through, Kompress fails to shrink (fallback_tokens NOT < compressed_tokens),
- # and — because the content is valid JSON and the log compressor is enabled —
- # the last-ditch Log fallback runs and shrinks. All of that is the shared,
- # unchanged block; the flip only feeds it identical entry state. Chain
- # [smart_crusher, kompress, log].
- router = _router()
- _isolate_branch(monkeypatch, router)
- monkeypatch.setattr(
- router,
- "_get_smart_crusher",
- lambda: SimpleNamespace(crush=lambda c, query="", bias=1.0: SimpleNamespace(compressed=c)),
- )
- # Kompress returns content unchanged with a huge token count -> NOT a shrink,
- # so the else branch (Log fallback) is taken.
- monkeypatch.setattr(router, "_try_ml_compressor", lambda c, ctx, q: (c, 10**9))
- monkeypatch.setattr(
- router,
- "_get_log_compressor",
- lambda: SimpleNamespace(
- compress=lambda c, bias=1.0: SimpleNamespace(compressed="LOG_FOLDED")
- ),
- )
- out, tokens, chain = router._apply_strategy_to_content(
- _JSON, CompressionStrategy.SMART_CRUSHER, "ctx", bias=1.0
- )
- assert out == "LOG_FOLDED"
- assert tokens == _estimate_tokens("LOG_FOLDED")
- assert chain == [
- CompressionStrategy.SMART_CRUSHER.value,
- CompressionStrategy.KOMPRESS.value,
- CompressionStrategy.LOG.value,
- ]
-
-
-# ─────────────── KOMPRESS / TEXT: flipped (registry, ML boundary) ──────────────
-
-
-def _fake_kompress(seen: dict[str, object]) -> SimpleNamespace:
- """In-memory Kompress model: records compress kwargs, no ONNX/HF."""
-
- def _compress(text: str, **kwargs: object) -> SimpleNamespace:
- seen.update(kwargs)
- # ``compressed_tokens`` is the model's OWN count (7), deliberately unequal
- # to ``_estimate_tokens`` of the output. After the flip the branch reports
- # ``_estimate_tokens(output.content)`` and DISCARDS this tuple value — the
- # single approved metric change — so the tests below assert the metric is
- # the estimate, not 7.
- return SimpleNamespace(compressed="KOMPRESSED::" + text, compressed_tokens=7)
-
- return SimpleNamespace(
- is_ready=lambda: True,
- ensure_background_load=lambda: None,
- compress=_compress,
- )
-
-
-def test_kompress_registry_dispatch_matches_direct(monkeypatch: pytest.MonkeyPatch) -> None:
- # KOMPRESS now dispatches through the registry "kompress" adapter, which
- # delegates to _try_ml_compressor with ``question`` forwarded via config.
- router = _router()
- _isolate_branch(monkeypatch, router)
- seen: dict[str, object] = {}
- monkeypatch.setattr(router, "_get_kompress", lambda: _fake_kompress(seen))
-
- content = "some plain text that the ML model would compress. " * 4
- out, tokens, chain = router._apply_strategy_to_content(
- content, CompressionStrategy.KOMPRESS, "ctx", question="my question", bias=1.0
- )
- # (a) CONTENT preserved — byte-identical to the direct _try_ml_compressor call
- # with the SAME question (so question is forwarded through the adapter).
- assert out == "KOMPRESSED::" + content
- assert chain == [CompressionStrategy.KOMPRESS.value]
- direct, direct_tokens = router._try_ml_compressor(content, "ctx", "my question")
- assert out == direct
- # The real ``question`` reached the model (config['question'] -> adapter ->
- # _try_ml_compressor -> compressor.compress), fixing the latent adapter bug
- # that dropped it (passed None).
- assert seen["question"] == "my question"
- # (b) APPROVED metric change: the token count is now _estimate_tokens(output),
- # NOT the model's own tuple value (7) that the direct call returns.
- assert tokens == _estimate_tokens(out)
- assert tokens != direct_tokens
- assert direct_tokens == 7
-
-
-def test_text_registry_dispatch_matches_direct(monkeypatch: pytest.MonkeyPatch) -> None:
- # TEXT shares the "kompress" adapter — same guarantees as the KOMPRESS branch.
- router = _router()
- _isolate_branch(monkeypatch, router)
- seen: dict[str, object] = {}
- monkeypatch.setattr(router, "_get_kompress", lambda: _fake_kompress(seen))
-
- content = "plain prose the text strategy sends straight to kompress. " * 4
- out, tokens, chain = router._apply_strategy_to_content(
- content, CompressionStrategy.TEXT, "ctx", question="q2", bias=1.0
- )
- assert out == "KOMPRESSED::" + content
- assert chain == [CompressionStrategy.TEXT.value]
- direct, direct_tokens = router._try_ml_compressor(content, "ctx", "q2")
- assert out == direct
- assert seen["question"] == "q2"
- assert tokens == _estimate_tokens(out)
- assert tokens != direct_tokens
- assert direct_tokens == 7
-
-
-def test_kompress_question_forwarding_changes_content(monkeypatch: pytest.MonkeyPatch) -> None:
- # QA-differential: a question-aware fake model embeds the question in its
- # output, so a DIFFERENT ``question`` yields DIFFERENT content. Proves the
- # router forwards ``question`` end-to-end via the registry adapter
- # (config['question'] -> _invoke_kompress -> _try_ml_compressor ->
- # compressor.compress) — the fix for the adapter that previously dropped it.
- router = _router()
- _isolate_branch(monkeypatch, router)
-
- def _qa_model() -> SimpleNamespace:
- def _compress(text: str, *, question: object = None, **kwargs: object) -> SimpleNamespace:
- return SimpleNamespace(compressed=f"Q[{question}]::{text}", compressed_tokens=5)
-
- return SimpleNamespace(
- is_ready=lambda: True,
- ensure_background_load=lambda: None,
- compress=_compress,
- )
-
- monkeypatch.setattr(router, "_get_kompress", _qa_model)
- content = "the body the model compresses conditioned on the question. " * 3
-
- out_a, _tok_a, _ = router._apply_strategy_to_content(
- content, CompressionStrategy.KOMPRESS, "ctx", question="alpha", bias=1.0
- )
- out_b, _tok_b, _ = router._apply_strategy_to_content(
- content, CompressionStrategy.KOMPRESS, "ctx", question="beta", bias=1.0
- )
- assert out_a != out_b
- assert out_a.startswith("Q[alpha]::")
- assert out_b.startswith("Q[beta]::")
- # Each matches the direct _try_ml_compressor call with the SAME question.
- assert out_a == router._try_ml_compressor(content, "ctx", "alpha")[0]
- assert out_b == router._try_ml_compressor(content, "ctx", "beta")[0]
+ assert json.loads(compressed) == records
+ if pretty:
+ assert chain == [CompressionStrategy.JSON_MINIFIER.value]
+ assert tokens < _estimate_tokens(content)
+ else:
+ assert compressed == content
diff --git a/tests/test_stateless_no_disk.py b/tests/test_stateless_no_disk.py
index 7b773a197..4092f60ac 100644
--- a/tests/test_stateless_no_disk.py
+++ b/tests/test_stateless_no_disk.py
@@ -30,6 +30,7 @@ from fastapi.testclient import TestClient
from headroom import paths
from headroom.cache import compression_store as cs
from headroom.cache.backends import InMemoryBackend
+from headroom.parser import CCR_RETRIEVAL_MARKER_RE
# The runtime log is the one file stateless mode is documented to keep writing
# (it is created owner-only and carries no request content by default).
@@ -82,15 +83,14 @@ def _snapshot(root: Path) -> set[str]:
def _tool_heavy_messages() -> list[dict]:
large = json.dumps(
[
- {
- "id": i,
- "name": f"Item {i}",
- "description": (
- f"Detailed description for item {i}: status active, created on "
- f"2024-01-{(i % 28) + 1:02d}, category=electronics, price={i * 10.99:.2f}."
- ),
- "tags": ["electronics", "sale", "featured"],
- }
+ # Text rows cannot take table compaction's lossless no-cache path.
+ # Repetition gives the real keep/drop transform and CCR something
+ # to exercise without relying on serialization-only token savings.
+ (
+ f"Detailed description for item {i % 20}: status active, created on "
+ f"2024-01-{(i % 20) + 1:02d}, category=electronics, "
+ f"price={(i % 20) * 10.99:.2f}. Tags: electronics, sale, featured."
+ )
for i in range(200)
]
)
@@ -123,12 +123,31 @@ def _run_turn(stateless: bool) -> None:
cost_tracking_enabled=False,
)
app = create_app(config)
+ messages = _tool_heavy_messages()
with TestClient(app, base_url="http://127.0.0.1", client=("127.0.0.1", 12345)) as client:
+ # The default sidecar mode deliberately disables CCR store writes.
resp = client.post(
- "/v1/compress", json={"messages": _tool_heavy_messages(), "model": "gpt-4"}
+ "/v1/compress",
+ json={"messages": messages, "model": "gpt-4", "config": {"mode": "ccr"}},
)
assert resp.status_code == 200, resp.text
- assert resp.json()["tokens_saved"] > 0, "turn must actually exercise the CCR store"
+ data = resp.json()
+ assert data["tokens_saved"] > 0, "turn must actually compress content"
+ original = json.loads(messages[2]["content"])
+ compressed, _ = json.JSONDecoder().raw_decode(data["messages"][2]["content"])
+ kept = [
+ row
+ for row in compressed
+ if isinstance(row, str) and not CCR_RETRIEVAL_MARKER_RE.search(row)
+ ]
+ assert len(kept) < len(original)
+ assert all(row in original for row in kept)
+ retrieved = [client.post("/v1/retrieve", json={"hash": key}) for key in data["ccr_hashes"]]
+ for response in retrieved:
+ assert response.status_code == 200, response.text
+ assert any(
+ json.loads(response.json()["original_content"]) == original for response in retrieved
+ ), "the emitted CCR hash must recover every original tool row"
# ---- end to end -----------------------------------------------------------
diff --git a/tests/test_transforms/test_content_router.py b/tests/test_transforms/test_content_router.py
index d714b1224..3733d3fdc 100644
--- a/tests/test_transforms/test_content_router.py
+++ b/tests/test_transforms/test_content_router.py
@@ -1463,10 +1463,12 @@ class TestSmartCrusherFallback:
),
)
- compressed, compressed_tokens, strategy_chain = router._apply_strategy_to_content(
- content,
- CompressionStrategy.SMART_CRUSHER,
- context="",
+ compressed, compressed_tokens, strategy_chain, _reversible = (
+ router._apply_strategy_to_content(
+ content,
+ CompressionStrategy.SMART_CRUSHER,
+ context="",
+ )
)
# Strategy chain must include smart_crusher
@@ -1508,10 +1510,12 @@ class TestSmartCrusherFallback:
lambda self: mock_crusher,
)
- compressed, compressed_tokens, strategy_chain = router._apply_strategy_to_content(
- content,
- CompressionStrategy.SMART_CRUSHER,
- context="",
+ compressed, compressed_tokens, strategy_chain, _reversible = (
+ router._apply_strategy_to_content(
+ content,
+ CompressionStrategy.SMART_CRUSHER,
+ context="",
+ )
)
# SmartCrusher should handle JSON directly
@@ -1566,10 +1570,12 @@ class TestSmartCrusherFallback:
),
)
- compressed, compressed_tokens, strategy_chain = router._apply_strategy_to_content(
- repetitive,
- CompressionStrategy.SMART_CRUSHER,
- context="",
+ compressed, compressed_tokens, strategy_chain, _reversible = (
+ router._apply_strategy_to_content(
+ repetitive,
+ CompressionStrategy.SMART_CRUSHER,
+ context="",
+ )
)
# The chain must include the requested strategy
@@ -1599,10 +1605,12 @@ class TestSmartCrusherFallback:
plain = "This is just plain text. " * 200
- compressed, compressed_tokens, strategy_chain = router._apply_strategy_to_content(
- plain,
- CompressionStrategy.CODE_AWARE,
- context="",
+ compressed, compressed_tokens, strategy_chain, _reversible = (
+ router._apply_strategy_to_content(
+ plain,
+ CompressionStrategy.CODE_AWARE,
+ context="",
+ )
)
# CodeAware should be in the chain
@@ -1829,7 +1837,7 @@ class TestMixedContentTagProtection:
router = ContentRouter(ContentRouterConfig(min_section_tokens=1))
def mangle(content, strategy, context, language=None, question=None, bias=1.0):
- return "MANGLED", 1, None
+ return "MANGLED", 1, [strategy.value], False
router._apply_strategy_to_content = mangle # type: ignore[method-assign]
return router
@@ -1865,3 +1873,23 @@ class TestMixedContentTagProtection:
assert "MANGLED" in result.compressed
assert result.strategy_used == CompressionStrategy.MIXED
+
+
+class TestMixedContentAttribution:
+ def test_minified_json_with_unchanged_code_remains_mixed(self):
+ value = {f"field_{i}": i for i in range(500)}
+ code = "print('preserve this unchanged')"
+ content = f"```python\n{code}\n```\n\n{json.dumps(value, indent=2)}"
+ router = ContentRouter(ContentRouterConfig(min_section_tokens=10, enable_kompress=False))
+
+ result = router._compress_mixed(content, context="")
+
+ assert result.strategy_used == CompressionStrategy.MIXED
+ assert code in result.compressed
+ json_block = result.compressed.split("```", 2)[2].strip()
+ assert json.loads(json_block) == value
+ assert {decision.strategy for decision in result.routing_log} == {
+ CompressionStrategy.CODE_AWARE,
+ CompressionStrategy.JSON_MINIFIER,
+ }
+ assert result.total_compressed_tokens < result.total_original_tokens
diff --git a/tests/test_transforms/test_smart_crusher_rust_parity.py b/tests/test_transforms/test_smart_crusher_rust_parity.py
index 79ccecebe..adb4fe93d 100644
--- a/tests/test_transforms/test_smart_crusher_rust_parity.py
+++ b/tests/test_transforms/test_smart_crusher_rust_parity.py
@@ -42,16 +42,32 @@ _FIXTURES_DIR = Path(__file__).parent.parent / "parity" / "fixtures" / "smart_cr
def _all_fixtures() -> list[Path]:
- return sorted(_FIXTURES_DIR.glob("*.json"))
-
-
-def test_at_least_17_fixtures_present():
- """Sanity check: the recorded fixture suite landed."""
- fixtures = _all_fixtures()
- assert len(fixtures) >= 17, (
- f"expected >= 17 fixtures, found {len(fixtures)}. "
- "If you re-recorded and got fewer, something deleted them."
- )
+ fixtures = sorted(_FIXTURES_DIR.glob("*.json"))
+ labels = {p.stem.rsplit("_", 1)[0] for p in fixtures}
+ # Require each native boundary and transformed-value scenario in the matrix;
+ # an arbitrary fixture count can hide a deleted scenario behind duplicates.
+ for label in (
+ "non_json_passthrough",
+ "dict_array_30",
+ "string_array_25",
+ "number_array_40_changepoint",
+ "mixed_array",
+ "dict_array_30_bias_high",
+ "dict_array_30_bias_low",
+ "dict_array_100_sequential",
+ "duplicate_dicts_40",
+ "empty_array",
+ "small_object_passthrough",
+ "short_array_passthrough",
+ "nulls_and_bools",
+ "unicode_dict_array",
+ "time_series_50",
+ "nested_object_with_array",
+ "nested_3deep_with_array",
+ ):
+ if label not in labels:
+ raise FileNotFoundError(f"Missing native parity scenario: {label}")
+ return fixtures
@pytest.mark.parametrize("fixture_path", _all_fixtures(), ids=lambda p: p.name)
diff --git a/tests/test_transforms_config_compressor.py b/tests/test_transforms_config_compressor.py
index 96d778c6a..56a30d566 100644
--- a/tests/test_transforms_config_compressor.py
+++ b/tests/test_transforms_config_compressor.py
@@ -572,19 +572,9 @@ def test_router_compresses_k8s_manifest_end_to_end() -> None:
def test_router_lossless_mode_uses_lossless_config_label() -> None:
router = ContentRouter(ContentRouterConfig(lossless=True))
- compressed, _tokens, chain = router._apply_strategy_to_content(
+ compressed, _tokens, chain, _reversible = router._apply_strategy_to_content(
K8S_MANIFEST, CompressionStrategy.CONFIG, context=""
)
assert chain == ["lossless_config"]
assert expand_runs(unfold_repeated_blocks(compressed)) == K8S_MANIFEST
assert not CCR_RETRIEVAL_MARKER_RE.search(compressed)
-
-
-def test_router_disabled_flag_skips_config_compressor() -> None:
- router = ContentRouter(ContentRouterConfig(enable_config_compressor=False))
- compressed, _tokens, _chain = router._apply_strategy_to_content(
- K8S_MANIFEST, CompressionStrategy.CONFIG, context=""
- )
- # Falls through to the unified fallback path; must not crash and must
- # not claim CONFIG did work it didn't do.
- assert isinstance(compressed, str)
diff --git a/tests/test_transforms_content_router.py b/tests/test_transforms_content_router.py
index 6c37a5a9c..3c1909dae 100644
--- a/tests/test_transforms_content_router.py
+++ b/tests/test_transforms_content_router.py
@@ -512,96 +512,6 @@ def test_force_kompress_apply_lightweight_detection_protects_recent_code(
assert result.transforms_applied == ["router:protected:recent_code"]
-def test_content_router_mixed_pure_apply_and_toin(monkeypatch: pytest.MonkeyPatch) -> None:
- router = ContentRouter()
- mixed_content = "\n".join(["before", "```python", "print('x')", "```", "after"])
- monkeypatch.setattr(
- content_router_module,
- "split_into_sections",
- lambda content, isolate=(): [
- SimpleNamespace(
- content="print('x')",
- content_type=ContentType.SOURCE_CODE,
- language="python",
- is_code_fence=True,
- ),
- SimpleNamespace(
- content="after text",
- content_type=ContentType.PLAIN_TEXT,
- language=None,
- is_code_fence=False,
- ),
- ],
- )
- monkeypatch.setattr(
- router,
- "_apply_strategy_to_content",
- lambda content, strategy, context, language=None, question=None, bias=1.0: (
- f"{strategy.value}:{content}",
- len(content.split()) - 1,
- [strategy.value],
- ),
- )
- result = router._compress_mixed(mixed_content, "ctx")
- assert result.strategy_used is CompressionStrategy.MIXED
- assert result.sections_processed == 2
- assert "```python\ncode_aware:print('x')\n```" in result.compressed
-
- monkeypatch.setattr(
- router,
- "_apply_strategy_to_content",
- lambda content, strategy, context, language=None, question=None, bias=1.0: (
- "shrunk",
- 1,
- [strategy.value],
- ),
- )
- pure = router._compress_pure("some plain text", CompressionStrategy.TEXT, "ctx")
- assert pure.routing_log[0].content_type is ContentType.PLAIN_TEXT
- assert pure.total_original_tokens == _estimate_tokens("some plain text")
- assert pure.total_compressed_tokens == 1
-
- calls: list[dict] = []
- router._toin = SimpleNamespace(record_compression=lambda **kwargs: calls.append(kwargs))
- monkeypatch.setattr(content_router_module, "_create_content_signature", lambda **kwargs: "sig")
- router._record_to_toin(
- CompressionStrategy.TEXT,
- "original content",
- "small",
- original_tokens=10,
- compressed_tokens=4,
- language="python",
- context="question",
- )
- assert calls[0]["tool_signature"] == "sig"
- assert calls[0]["strategy"] == "text"
- assert calls[0]["query_context"] == "question"
-
- router._record_to_toin(
- CompressionStrategy.SMART_CRUSHER,
- "x",
- "x",
- original_tokens=10,
- compressed_tokens=4,
- )
- router._record_to_toin(
- CompressionStrategy.TEXT,
- "x",
- "x",
- original_tokens=2,
- compressed_tokens=2,
- )
- monkeypatch.setattr(content_router_module, "_create_content_signature", lambda **kwargs: None)
- router._record_to_toin(
- CompressionStrategy.TEXT,
- "x",
- "y",
- original_tokens=5,
- compressed_tokens=1,
- )
- assert len(calls) == 1
-
-
def test_diff_strategy_does_not_fallback_to_kompress_when_diff_is_noop(
monkeypatch: pytest.MonkeyPatch,
) -> None:
@@ -619,7 +529,7 @@ def test_diff_strategy_does_not_fallback_to_kompress_when_diff_is_noop(
monkeypatch.setattr(router, "_try_ml_compressor", fail_kompress)
- compressed, compressed_tokens, strategy_chain = router._apply_strategy_to_content(
+ compressed, compressed_tokens, strategy_chain, _reversible = router._apply_strategy_to_content(
diff,
CompressionStrategy.DIFF,
context="",
@@ -647,7 +557,7 @@ def test_log_strategy_does_not_fallback_to_kompress_when_log_is_noop(
monkeypatch.setattr(router, "_try_ml_compressor", fail_kompress)
- compressed, compressed_tokens, strategy_chain = router._apply_strategy_to_content(
+ compressed, compressed_tokens, strategy_chain, _reversible = router._apply_strategy_to_content(
log,
CompressionStrategy.LOG,
context="",
@@ -693,7 +603,7 @@ def test_smart_crusher_log_fallback_skipped_for_invalid_json(
lambda content, context, question=None: (content, len(content.split())),
)
- compressed, compressed_tokens, strategy_chain = router._apply_strategy_to_content(
+ compressed, compressed_tokens, strategy_chain, _reversible = router._apply_strategy_to_content(
truncated,
CompressionStrategy.SMART_CRUSHER,
context="",
@@ -1077,49 +987,6 @@ def test_relevance_split_does_not_fragment_json_payloads(
assert kompress_inputs == []
-def test_smart_crusher_log_fallback_runs_for_valid_json(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- """Valid JSON arrays still reach the Log fallback when SmartCrusher no-ops.
-
- The JSON-validity guard (#1306) only blocks the Log fallback for
- *invalid* JSON. Valid JSON arrays that SmartCrusher can't shrink (e.g.
- repetitive JSONL the proxy's own log dumps) still get the Log fallback,
- which is the intended "repetitive JSONL" path.
- """
- router = ContentRouter(ContentRouterConfig())
- valid_json = '[{"a": 1}, {"a": 1}, {"a": 1}, {"a": 1}]'
-
- class NoopSmartCrusher:
- def crush(self, content: str, query: str = "", bias: float = 1.0) -> SimpleNamespace:
- return SimpleNamespace(compressed=content)
-
- class ShrinkingLogCompressor:
- def compress(self, content: str, bias: float = 1.0) -> SimpleNamespace:
- return SimpleNamespace(compressed="[compressed]")
-
- monkeypatch.setattr(router, "_get_smart_crusher", lambda: NoopSmartCrusher())
- monkeypatch.setattr(router, "_get_log_compressor", lambda: ShrinkingLogCompressor())
- # Kompress no-op → Log fallback fires. A faithful no-op reports the same
- # token count the router computed for the (unchanged) content — using
- # _estimate_tokens, not a naive word split, so it isn't mistaken for a
- # saving once #1857's whitespace-aware counting rates the JSON above 8.
- monkeypatch.setattr(
- router,
- "_try_ml_compressor",
- lambda content, context, question=None: (content, _estimate_tokens(content)),
- )
-
- compressed, _compressed_tokens, strategy_chain = router._apply_strategy_to_content(
- valid_json,
- CompressionStrategy.SMART_CRUSHER,
- context="",
- )
-
- assert CompressionStrategy.LOG.value in strategy_chain
- assert compressed == "[compressed]"
-
-
# ---------------------------------------------------------------------------
# MIXED false-positive on source code: Python with dict literals + docstrings
# triggers ``has_json_blocks`` and ``has_prose`` in ``is_mixed_content``,
@@ -2633,41 +2500,34 @@ def test_html_with_embedded_json_runs_extractor_not_json_shortcut() -> None:
assert len(result.compressed) < len(doc) * 3 // 5
-def test_html_extraction_miss_still_gets_deferred_json_splice() -> None:
- """Deferring must not LOSE the embedded-JSON win when extraction finds nothing.
-
- A shell-only HTML document makes the extractor come back empty; the
- deferred splice then still compresses the embedded array. Before the fix
- this shape short-circuited at the top (chain ``["embedded_json"]``); now
- the block first proves the HTML strategy is a miss (chain
- ``["html", "embedded_json"]``) with the same spliced output.
- """
- pytest.importorskip("trafilatura")
- embedded = json.dumps(
- [
- {"id": i, "name": f"item-{i}", "tags": ["alpha", "beta"], "score": i * 3}
- for i in range(40)
- ]
+def test_json_in_html_preserves_records_and_markup() -> None:
+ rows = [
+ {"id": i, "account": f"ACCOUNT_{i}", "score": i * 13, "active": i % 2 == 0}
+ for i in range(60)
+ ]
+ prefix = (
+ "<!DOCTYPE html>\n<html><head><title>data</title></head>\n<body>\n"
+ '<script type="application/json">'
)
- shell = (
- "<!DOCTYPE html>\n<html><head><title>t</title></head>\n<body>\n"
- '<script type="application/json">' + embedded + "</script>\n"
- "</body></html>\n"
+ suffix = "</script>\n</body></html>\n"
+ shell = prefix + json.dumps(rows) + suffix
+ router = ContentRouter(
+ ContentRouterConfig(
+ lossless=False,
+ enable_html_extractor=True,
+ smart_crusher_lossless_only=True,
+ smart_crusher_with_compaction=False,
+ enable_kompress=False,
+ )
)
- router = ContentRouter()
- # sanity: extraction finds nothing on this shell (empty or None)
- extractor = router._get_html_extractor()
- assert extractor is not None
- assert not (extractor.extract(shell).extracted or "").strip()
- text, _tokens, chain = router._apply_strategy_to_content(
- shell, CompressionStrategy.HTML, "", None, 1.0
- )
+ result = router.compress(shell)
- assert chain == ["html", "embedded_json"]
- assert len(text) < len(shell)
- json_marker = '"id":'
- assert json_marker not in text # the array was compressed away
+ assert result.compressed.startswith(prefix)
+ assert result.compressed.endswith(suffix)
+ assert json.loads(result.compressed[len(prefix) : -len(suffix)]) == rows
+ assert result.tokens_saved > 0
+ assert not result.unrecoverable_sections
# A Claude Code Read of Rust: line-numbered code that code_aware hands back
@@ -2713,7 +2573,7 @@ def test_code_aware_noop_keeps_a_smaller_kompress_fallback(
assert _estimate_tokens(kompressed) > len(_NUMBERED_RUST.split())
router = _noop_code_aware_router(monkeypatch, kompressed)
- compressed, compressed_tokens, strategy_chain = router._apply_strategy_to_content(
+ compressed, compressed_tokens, strategy_chain, _reversible = router._apply_strategy_to_content(
_NUMBERED_RUST, CompressionStrategy.CODE_AWARE, context=""
)
@@ -2728,7 +2588,7 @@ def test_code_aware_noop_reports_tokens_when_kompress_passes_through(
"""An unchanged block is reported at its own token count, never its word count."""
router = _noop_code_aware_router(monkeypatch, _NUMBERED_RUST)
- compressed, compressed_tokens, _chain = router._apply_strategy_to_content(
+ compressed, compressed_tokens, _chain, _reversible = router._apply_strategy_to_content(
_NUMBERED_RUST, CompressionStrategy.CODE_AWARE, context=""
)
@@ -2751,7 +2611,7 @@ def test_code_aware_noop_runs_kompress_once_when_it_passes_through(
monkeypatch.setattr(router, "_try_ml_compressor", counting)
- compressed, compressed_tokens, _chain = router._apply_strategy_to_content(
+ compressed, compressed_tokens, _chain, _reversible = router._apply_strategy_to_content(
_NUMBERED_RUST, CompressionStrategy.CODE_AWARE, context=""
)
@@ -2780,7 +2640,7 @@ def test_code_aware_noop_keeps_a_smaller_inline_kompress(
monkeypatch.setattr(router, "_try_ml_compressor", counting)
- compressed, compressed_tokens, strategy_chain = router._apply_strategy_to_content(
+ compressed, compressed_tokens, strategy_chain, _reversible = router._apply_strategy_to_content(
_NUMBERED_RUST, CompressionStrategy.CODE_AWARE, context=""
)
@@ -2788,3 +2648,138 @@ def test_code_aware_noop_keeps_a_smaller_inline_kompress(
assert compressed_tokens == _estimate_tokens(kompressed)
assert strategy_chain == ["code_aware", "kompress"]
assert len(calls) == 1
+
+
[email protected]("as_block", [False, True], ids=["string-tool", "tool-result-block"])
[email protected]("other_marker", [False, True], ids=["no-marker", "unrelated-marker"])
[email protected]("ccr_enabled", [False, True], ids=["explicit-no-ccr", "reversible-tools"])
+def test_mixed_lossy_section_cannot_borrow_another_sections_retrieval_marker(
+ monkeypatch, as_block, other_marker, ccr_enabled
+) -> None:
+ router = ContentRouter(
+ ContentRouterConfig(
+ enable_code_aware=True,
+ enable_kompress=False,
+ min_chars_for_block_compression=0,
+ protect_recent_code=0,
+ protect_analysis_context=False,
+ ccr_inject_marker=ccr_enabled,
+ )
+ )
+ code = (
+ "def account():\n"
+ + "".join(
+ f" # audit record {i}: preserve the complete account history.\n" for i in range(120)
+ )
+ + ' identifier = "ACCOUNT_ID_42"\n return identifier\n'
+ )
+ content = "```python\n" + code + "```\n\n" + '[{"flag":true,"value":7}]'
+ if other_marker:
+ content += "\n\nIndependent retained section <<ccr:123456789012>>"
+
+ def compress_section(section, strategy, *_args, **_kwargs):
+ # Control only the lossy compressor boundary; split/routing/guards are real.
+ compressed = (
+ section.replace("ACCOUNT_ID_42", "")
+ if strategy is CompressionStrategy.CODE_AWARE
+ else section
+ )
+ return compressed, _estimate_tokens(compressed), [strategy.value], compressed == section
+
+ monkeypatch.setattr(router, "_apply_strategy_to_content", compress_section)
+ monkeypatch.setattr(
+ router,
+ "compress",
+ lambda c, context="", bias=1.0, precomputed_detection=None: router._compress_mixed(
+ c, context, bias=bias
+ ),
+ )
+ direct = router.compress(content)
+ assert direct.strategy_used is CompressionStrategy.MIXED
+ assert direct.compression_ratio < 1.0
+ assert "ACCOUNT_ID_42" not in direct.compressed
+
+ message = (
+ _tool_result_block_msg(content)
+ if as_block
+ else {"role": "tool", "tool_call_id": "account", "content": content}
+ )
+ for _turn in range(2):
+ result = router.apply([message], _ChurnTokenizer(), model_limit=100_000)
+ output = (
+ result.messages[0]["content"][0]["content"]
+ if as_block
+ else result.messages[0]["content"]
+ )
+ if ccr_enabled or other_marker:
+ assert output == content
+ else:
+ assert "ACCOUNT_ID_42" not in output
+ assert '[{"flag":true,"value":7}]' in output
+
+
[email protected]("as_block", [False, True], ids=["string-tool", "tool-result-block"])
[email protected]("mode", ["minify", "unmarked-loss", "sibling-marker"])
+def test_embedded_json_mixed_tool_output_preserves_every_record(
+ monkeypatch: pytest.MonkeyPatch, as_block: bool, mode: str
+) -> None:
+ from headroom.providers import OpenAIProvider
+ from headroom.tokenizer import Tokenizer
+
+ monkeypatch.setenv("HEADROOM_DETECT_BACKEND", "python")
+ router = ContentRouter(
+ ContentRouterConfig(
+ smart_crusher_with_compaction=False,
+ smart_crusher_max_items_after_crush=1000,
+ enable_kompress=False,
+ lossless_then_lossy=False,
+ relevance_split=False,
+ min_chars_for_block_compression=0,
+ protect_recent_code=0,
+ protect_analysis_context=False,
+ ccr_inject_marker=True,
+ )
+ )
+ rows = [{"id": i, "account": f"ACCOUNT_ID_{i}", "value": i * 13} for i in range(60)]
+ fenced = "\n".join(f"Retain the complete diagnostic note for customer {i}." for i in range(30))
+ content = f"```text\n{fenced}\n```\n\n"
+ if mode == "sibling-marker":
+ sibling = json.dumps([{"recover": i} for i in range(20)])
+ content += f"Independent: {sibling}\n\n"
+ content += f"Records: {json.dumps(rows)}\n\nDone."
+ if mode != "minify":
+ # Control the lossy compressor only. Nested dispatch, span adoption,
+ # mixed routing, both safety guards and their caches remain production code.
+ def crush(span, **_kwargs):
+ values = json.loads(span)
+ text = (
+ "<<ccr:123456789012>>"
+ if "recover" in values[0]
+ else json.dumps(values[:1], separators=(",", ":"))
+ )
+ return SimpleNamespace(compressed=text)
+
+ monkeypatch.setattr(router, "_get_smart_crusher", lambda: SimpleNamespace(crush=crush))
+
+ tokenizer = Tokenizer(OpenAIProvider().get_token_counter("gpt-4o"), "gpt-4o")
+ message = (
+ _tool_result_block_msg(content)
+ if as_block
+ else {"role": "tool", "tool_call_id": "records", "content": content}
+ )
+ for _turn in range(2): # Cold compression followed by the cached/skip verdict.
+ result = router.apply([message], tokenizer, model_limit=100_000)
+ output = (
+ result.messages[0]["content"][0]["content"]
+ if as_block
+ else result.messages[0]["content"]
+ )
+ if mode == "minify":
+ assert tokenizer.count_text(output) < tokenizer.count_text(content)
+ assert f"```text\n{fenced}\n```" in output
+ assert output.endswith("Done.")
+ assert "<<ccr:" not in output
+ recovered, _end = json.JSONDecoder().raw_decode(output.split("Records: ", 1)[1])
+ assert recovered == rows
+ else:
+ assert output == content