Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

chore(rust): SmartCrusher CCR marker injection + walker unification

chopratejasda7716a95af0
diff --git a/crates/headroom-core/src/transforms/smart_crusher/compaction/mod.rs b/crates/headroom-core/src/transforms/smart_crusher/compaction/mod.rs
index 5b94dd434..cecfe60d7 100644
--- a/crates/headroom-core/src/transforms/smart_crusher/compaction/mod.rs
+++ b/crates/headroom-core/src/transforms/smart_crusher/compaction/mod.rs
@@ -33,7 +33,9 @@ pub use classifier::{classify_cell, CellClass, ClassifyConfig};
pub use compactor::{compact, CompactConfig};
pub use formatter::{CsvSchemaFormatter, Formatter, JsonFormatter};
pub use ir::{Bucket, CellValue, Compaction, FieldSpec, OpaqueKind, Row, Schema};
-pub use walker::{compact_document, DocumentCompactor};
+pub use walker::{
+ compact_document, emit_opaque_ccr_marker, try_parse_json_container, DocumentCompactor,
+};
/// Composed compaction stage: a config + formatter pair.
///
diff --git a/crates/headroom-core/src/transforms/smart_crusher/compaction/walker.rs b/crates/headroom-core/src/transforms/smart_crusher/compaction/walker.rs
index 59e8631c1..1d6672a2e 100644
--- a/crates/headroom-core/src/transforms/smart_crusher/compaction/walker.rs
+++ b/crates/headroom-core/src/transforms/smart_crusher/compaction/walker.rs
@@ -29,12 +29,15 @@
//! BEFORE running TabularCompactor on the array, so inner sub-tables
//! become strings first and the outer table sees them as cells.
+use std::sync::Arc;
+
use serde_json::{Map, Value};
use super::classifier::{classify_cell, CellClass};
use super::compactor::{compact, CompactConfig};
use super::formatter::{CsvSchemaFormatter, Formatter};
use super::ir::OpaqueKind;
+use crate::ccr::CcrStore;
use sha2::{Digest, Sha256};
@@ -50,6 +53,10 @@ use sha2::{Digest, Sha256};
pub struct DocumentCompactor {
pub config: CompactConfig,
pub formatter: Box<dyn Formatter>,
+ /// Optional CCR store. When set, opaque-string CCR markers also
+ /// stash the original blob keyed by the marker hash, mirroring the
+ /// row-drop CCR contract from `SmartCrusher::crush_array`.
+ pub ccr_store: Option<Arc<dyn CcrStore>>,
}
impl Default for DocumentCompactor {
@@ -57,6 +64,7 @@ impl Default for DocumentCompactor {
Self {
config: CompactConfig::default(),
formatter: Box::new(CsvSchemaFormatter::new()),
+ ccr_store: None,
}
}
}
@@ -76,6 +84,11 @@ impl DocumentCompactor {
self
}
+ pub fn with_ccr_store(mut self, store: Arc<dyn CcrStore>) -> Self {
+ self.ccr_store = Some(store);
+ self
+ }
+
/// Walk and compact. Returns a JSON value with the same shape but
/// with compactable spots replaced by rendered strings.
pub fn compact(&self, doc: Value) -> Value {
@@ -124,10 +137,11 @@ fn walk_string(s: String, ctx: &DocumentCompactor) -> Value {
};
}
- // Long opaque blob: substitute with CCR marker.
+ // Long opaque blob: substitute with CCR marker (and stash the
+ // original in the store if one is configured, so retrieval works).
if let CellClass::Opaque(kind) = classify_cell(&Value::String(s.clone()), &ctx.config.classify)
{
- return Value::String(format_ccr_marker(s.as_bytes(), &kind));
+ return Value::String(emit_opaque_ccr_marker(&s, &kind, ctx.ccr_store.as_ref()));
}
Value::String(s)
@@ -136,7 +150,7 @@ fn walk_string(s: String, ctx: &DocumentCompactor) -> Value {
/// Parse a string as JSON IF it looks like a container (starts with `{`
/// or `[`) AND parses cleanly to Object/Array. Returns None otherwise —
/// we don't recurse on bare scalars even if they parse.
-fn try_parse_json_container(s: &str) -> Option<Value> {
+pub fn try_parse_json_container(s: &str) -> Option<Value> {
let trimmed = s.trim_start();
if !matches!(trimmed.chars().next(), Some('{') | Some('[')) {
return None;
@@ -146,22 +160,37 @@ fn try_parse_json_container(s: &str) -> Option<Value> {
.filter(|v| matches!(v, Value::Object(_) | Value::Array(_)))
}
-fn format_ccr_marker(bytes: &[u8], kind: &OpaqueKind) -> String {
+/// Emit an opaque-blob CCR marker AND (optionally) stash the original
+/// in the store so retrieval works. The hash is computed identically
+/// regardless of store presence — same input → same marker — so the
+/// runtime contract is stable across configurations.
+///
+/// Marker format: `<<ccr:HASH,KIND,SIZE>>` where HASH is the 12-char
+/// SHA-256 hex prefix of the payload bytes, KIND is `base64` / `string`
+/// / `html` / custom, SIZE is humanized (`123B`, `4.5KB`, `1.2MB`).
+pub fn emit_opaque_ccr_marker(
+ payload: &str,
+ kind: &OpaqueKind,
+ store: Option<&Arc<dyn CcrStore>>,
+) -> String {
let mut h = Sha256::new();
- h.update(bytes);
+ h.update(payload.as_bytes());
let hash: String = h
.finalize()
.iter()
.take(6)
.map(|b| format!("{b:02x}"))
.collect();
+ if let Some(s) = store {
+ s.put(&hash, payload);
+ }
let kind_str = match kind {
OpaqueKind::Base64Blob => "base64",
OpaqueKind::LongString => "string",
OpaqueKind::HtmlChunk => "html",
OpaqueKind::Other(s) => s.as_str(),
};
- format!("<<ccr:{},{},{}>>", hash, kind_str, humanize(bytes.len()))
+ format!("<<ccr:{},{},{}>>", hash, kind_str, humanize(payload.len()))
}
fn humanize(n: usize) -> String {
diff --git a/crates/headroom-core/src/transforms/smart_crusher/crusher.rs b/crates/headroom-core/src/transforms/smart_crusher/crusher.rs
index cf9739442..7b4a99654 100644
--- a/crates/headroom-core/src/transforms/smart_crusher/crusher.rs
+++ b/crates/headroom-core/src/transforms/smart_crusher/crusher.rs
@@ -40,9 +40,10 @@ use serde_json::Value;
use super::analyzer::SmartAnalyzer;
use super::builder::SmartCrusherBuilder;
use super::classifier::{classify_array, ArrayType};
-use super::compaction::classifier::{classify_cell, CellClass, ClassifyConfig};
-use super::compaction::ir::OpaqueKind;
-use super::compaction::{Compaction, CompactionStage};
+use super::compaction::{
+ classify_cell, emit_opaque_ccr_marker, try_parse_json_container, CellClass, ClassifyConfig,
+ Compaction, CompactionStage,
+};
use super::config::SmartCrusherConfig;
use super::crushers::{compute_k_split, crush_number_array, crush_object, crush_string_array};
use super::planning::SmartCrusherPlanner;
@@ -390,7 +391,31 @@ impl SmartCrusher {
n,
result.items.len()
));
- return (Value::Array(result.items), info_parts.join(","));
+ // Lossy path with rows dropped → append a
+ // CCR-Dropped sentinel object as the last
+ // element of the kept-items array. This is
+ // the **only** place the LLM sees the
+ // `<<ccr:HASH ...>>` pointer in the prompt.
+ // Without this, the store has the data but
+ // no model can ever ask for it.
+ //
+ // Sentinel shape: `{"_ccr_dropped":
+ // "<<ccr:HASH N_rows_offloaded>>"}` —
+ // preserves "array-of-objects" shape so
+ // downstream consumers iterating with
+ // `x.get(...)` keep working; the well-known
+ // `_ccr_dropped` key signals metadata
+ // unambiguously.
+ let mut items = result.items;
+ if !result.dropped_summary.is_empty() {
+ let mut sentinel = serde_json::Map::new();
+ sentinel.insert(
+ "_ccr_dropped".to_string(),
+ Value::String(result.dropped_summary),
+ );
+ items.push(Value::Object(sentinel));
+ }
+ return (Value::Array(items), info_parts.join(","));
}
ArrayType::StringArray => {
let strs: Vec<&str> = arr.iter().filter_map(|v| v.as_str()).collect();
@@ -451,11 +476,10 @@ impl SmartCrusher {
(Value::Object(processed), info_parts.join(","))
}
- // Strings: walker-equivalent handling. Parse stringified-JSON
- // containers and recurse; CCR-substitute long opaque blobs.
- // (Stage 3c.2 PR5: brings DocumentCompactor's String semantics
- // into the public `crush()` path so tool outputs that wrap
- // JSON in strings, or contain opaque blobs, get processed.)
+ // Strings: walker-equivalent handling. Delegates to
+ // `process_string` which parses stringified-JSON containers
+ // (recursing through `process_value`) and CCR-substitutes
+ // opaque blobs (with store-write so retrieval works).
Value::String(s) => self.process_string(s, depth, query_context, bias),
// Other scalars — passthrough.
_ => (value.clone(), String::new()),
@@ -486,7 +510,7 @@ impl SmartCrusher {
bias: f64,
) -> (Value, String) {
// 1. Stringified-JSON: parse, recurse, re-render.
- if let Some(parsed) = try_parse_json_container_str(s) {
+ if let Some(parsed) = try_parse_json_container(s) {
let (processed, sub_info) = self.process_value(&parsed, depth + 1, query_context, bias);
// If recursion produced something different, re-emit.
// Special case: if the recursion returned a `Value::String`
@@ -509,14 +533,14 @@ impl SmartCrusher {
}
}
- // 2. Opaque blob: substitute with CCR marker.
+ // 2. Opaque blob: substitute with CCR marker AND stash the
+ // original in the store (PR8) so retrieval works. Hash + format
+ // identical to walker.rs via the shared helper — zero drift.
let cfg = ClassifyConfig::default();
if let CellClass::Opaque(kind) = classify_cell(&Value::String(s.to_string()), &cfg) {
- let marker = ccr_marker_for_string(s.as_bytes(), &kind);
- return (
- Value::String(marker),
- format!("string_ccr:{}", opaque_kind_label(&kind)),
- );
+ let marker = emit_opaque_ccr_marker(s, &kind, self.ccr_store.as_ref());
+ let kind_label = opaque_kind_label(&kind);
+ return (Value::String(marker), format!("string_ccr:{kind_label}"));
}
// 3. Plain string — passthrough.
@@ -890,44 +914,19 @@ fn hash_array_for_ccr(items: &[Value]) -> String {
.collect()
}
-// ─── PR5 walker-integration helpers (string handling) ───────────────────────
-
-/// Cheap parse-as-JSON-container check. Returns `Some(parsed)` only
-/// when `s` parses to an object or array — not for bare scalars
-/// (numbers, bools, nulls) that happen to be valid JSON literals.
-fn try_parse_json_container_str(s: &str) -> Option<Value> {
- let trimmed = s.trim_start();
- if !matches!(trimmed.chars().next(), Some('{') | Some('[')) {
- return None;
- }
- serde_json::from_str::<Value>(s)
- .ok()
- .filter(|v| matches!(v, Value::Object(_) | Value::Array(_)))
-}
-
-/// Format a CCR retrieval marker for an opaque string. Format must
-/// match `compaction::walker::format_ccr_marker` so downstream
-/// consumers can pattern-match markers regardless of which path
-/// emitted them.
-fn ccr_marker_for_string(bytes: &[u8], kind: &OpaqueKind) -> String {
- use sha2::{Digest, Sha256};
- let mut h = Sha256::new();
- h.update(bytes);
- let hash: String = h
- .finalize()
- .iter()
- .take(6)
- .map(|b| format!("{b:02x}"))
- .collect();
- format!(
- "<<ccr:{},{},{}>>",
- hash,
- opaque_kind_label(kind),
- humanize_bytes(bytes.len())
- )
-}
-
-fn opaque_kind_label(kind: &OpaqueKind) -> &str {
+// ─── PR5 walker-integration helpers (string handling) ──────────────────────
+//
+// Parse-as-JSON-container, marker formatting, and humanize-bytes used to
+// live here as locals. PR8 extracted them into `compaction::walker` so
+// `walker.rs` and `process_value` share one canonical implementation —
+// killing the drift risk where the two paths could format markers
+// differently. `process_string` now calls `try_parse_json_container` and
+// `emit_opaque_ccr_marker` directly. Only `opaque_kind_label` survives
+// here because `process_string`'s `string_ccr:<kind>` strategy-info
+// label is local to this module's debug-string convention.
+
+fn opaque_kind_label(kind: &super::compaction::OpaqueKind) -> &str {
+ use super::compaction::OpaqueKind;
match kind {
OpaqueKind::Base64Blob => "base64",
OpaqueKind::LongString => "string",
@@ -936,17 +935,6 @@ fn opaque_kind_label(kind: &OpaqueKind) -> &str {
}
}
-fn humanize_bytes(n: usize) -> String {
- if n < 1024 {
- return format!("{n}B");
- }
- let kb = n as f64 / 1024.0;
- if kb < 1024.0 {
- return format!("{kb:.1}KB");
- }
- format!("{:.1}MB", kb / 1024.0)
-}
-
#[cfg(test)]
mod tests {
use super::*;
@@ -1423,11 +1411,11 @@ mod tests {
#[test]
fn process_string_helper_parses_only_containers() {
- assert!(try_parse_json_container_str("{\"a\":1}").is_some());
- assert!(try_parse_json_container_str("[1,2,3]").is_some());
- assert!(try_parse_json_container_str("123").is_none()); // bare scalar
- assert!(try_parse_json_container_str("\"hello\"").is_none()); // bare string
- assert!(try_parse_json_container_str("not json").is_none());
- assert!(try_parse_json_container_str("{malformed").is_none());
+ assert!(try_parse_json_container("{\"a\":1}").is_some());
+ assert!(try_parse_json_container("[1,2,3]").is_some());
+ assert!(try_parse_json_container("123").is_none()); // bare scalar
+ assert!(try_parse_json_container("\"hello\"").is_none()); // bare string
+ assert!(try_parse_json_container("not json").is_none());
+ assert!(try_parse_json_container("{malformed").is_none());
}
}
diff --git a/crates/headroom-core/tests/ccr_roundtrip.rs b/crates/headroom-core/tests/ccr_roundtrip.rs
index 09b140451..28d35dd8a 100644
--- a/crates/headroom-core/tests/ccr_roundtrip.rs
+++ b/crates/headroom-core/tests/ccr_roundtrip.rs
@@ -222,3 +222,159 @@ fn full_crush_pipeline_roundtrips_through_store() {
let store_len = store.len();
assert!(store_len > 0, "expected at least one CCR store entry");
}
+
+// ─── PR8 additions: marker injection + walker unification ──────────
+
+#[test]
+fn lossy_crush_injects_marker_into_output_json() {
+ // Cornerstone of PR8: the public `crush()` API output now carries
+ // the `<<ccr:HASH ...>>` marker as a string element of the array.
+ // Without this, the LLM never sees the retrieval pointer.
+ let crusher = SmartCrusher::new(force_lossy_config());
+ let items = lossy_friendly_items(50);
+ let raw = serde_json::to_string(&Value::Array(items)).unwrap();
+
+ let result = crusher.crush(&raw, "", 1.0);
+
+ assert!(
+ result.compressed.contains("<<ccr:"),
+ "expected marker in output, got: {}",
+ result.compressed
+ );
+ assert!(
+ result.compressed.contains("rows_offloaded>>"),
+ "marker should advertise dropped count: {}",
+ result.compressed
+ );
+
+ // The hash in the marker is the same one in the store.
+ let marker_hash =
+ extract_hash_from_marker(&result.compressed).expect("marker must embed a hash");
+ let store = crusher.ccr_store().unwrap();
+ assert!(
+ store.get(&marker_hash).is_some(),
+ "marker hash {marker_hash} must resolve in the store"
+ );
+}
+
+#[test]
+fn nested_array_inside_object_gets_marker_injected() {
+ // The public crush() recurses through wrapper objects. A nested
+ // `events: [...]` array that triggers lossy must get a marker too.
+ let crusher = SmartCrusher::new(force_lossy_config());
+ let doc = json!({
+ "user": "alice",
+ "events": (0..50).map(|i| json!({"id": i, "status": "ok"}))
+ .collect::<Vec<_>>(),
+ });
+
+ let result = crusher.crush(&doc.to_string(), "", 1.0);
+ assert!(result.compressed.contains("<<ccr:"));
+
+ let parsed: Value = serde_json::from_str(&result.compressed).unwrap();
+ let events = parsed.get("events").expect("events field preserved");
+ let arr = events.as_array().expect("events stays an array");
+ let last = arr.last().expect("non-empty array");
+ let marker = last
+ .get("_ccr_dropped")
+ .and_then(|v| v.as_str())
+ .expect("sentinel object with _ccr_dropped key");
+ assert!(
+ marker.starts_with("<<ccr:"),
+ "expected CCR marker text, got: {marker}"
+ );
+}
+
+#[test]
+fn opaque_string_in_object_emits_marker_and_stores_original() {
+ // Walker semantics in process_value: a long base64-ish blob in
+ // an object field should be replaced with a CCR marker AND the
+ // original bytes stashed in the store.
+ let crusher = SmartCrusher::new(SmartCrusherConfig::default());
+ let store = crusher.ccr_store().unwrap().clone();
+ let starting_len = store.len();
+
+ // 64-char base64 alphabet repeated → tripping the opaque-blob
+ // detector deterministically.
+ let big = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=".repeat(8);
+ let doc = json!({"id": 1, "blob": big.clone()});
+
+ let result = crusher.crush(&doc.to_string(), "", 1.0);
+ let parsed: Value = serde_json::from_str(&result.compressed).unwrap();
+ let blob_out = parsed.get("blob").and_then(|v| v.as_str()).unwrap();
+
+ assert!(
+ blob_out.starts_with("<<ccr:") && blob_out.contains(",base64,"),
+ "expected base64 CCR marker, got: {blob_out}"
+ );
+
+ // The store grew by 1 and holds the original payload.
+ assert_eq!(
+ store.len(),
+ starting_len + 1,
+ "opaque-string CCR must write to store"
+ );
+ let marker_hash = extract_inner_hash(blob_out).unwrap();
+ let retrieved = store.get(&marker_hash).expect("hash must resolve");
+ assert_eq!(retrieved, big);
+}
+
+#[test]
+fn stringified_json_array_recurses_and_compacts() {
+ // A field whose value is a JSON-encoded array should be parsed,
+ // recursively crushed, and re-encoded — the walker behavior, now
+ // available from the main pipeline.
+ let crusher = SmartCrusher::new(force_lossy_config());
+ let inner = (0..50)
+ .map(|i| json!({"id": i, "status": "ok"}))
+ .collect::<Vec<_>>();
+ let inner_json = serde_json::to_string(&inner).unwrap();
+ let doc = json!({"id": "outer", "payload": inner_json});
+
+ let result = crusher.crush(&doc.to_string(), "", 1.0);
+ let parsed: Value = serde_json::from_str(&result.compressed).unwrap();
+ let payload = parsed.get("payload").and_then(|v| v.as_str()).unwrap();
+
+ // The payload string went through process_value's String arm,
+ // which parsed it as JSON, recursed, and re-emitted. Result:
+ // either a marker-bearing array string or a direct-rendered form.
+ assert!(
+ payload.contains("<<ccr:") || payload.contains("rows_offloaded"),
+ "expected stringified-JSON to be processed, got: {payload}"
+ );
+}
+
+#[test]
+fn document_walker_with_store_roundtrips_opaque_blob() {
+ // Direct DocumentCompactor usage with the same store.
+ use headroom_core::transforms::smart_crusher::compaction::DocumentCompactor;
+
+ let store: Arc<dyn CcrStore> = Arc::new(InMemoryCcrStore::new());
+ let dc = DocumentCompactor::new().with_ccr_store(store.clone());
+
+ let big = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=".repeat(8);
+ let out = dc.compact(json!({"id": 1, "blob": big.clone()}));
+ let blob = out.pointer("/blob").and_then(|v| v.as_str()).unwrap();
+ assert!(blob.starts_with("<<ccr:"));
+
+ let h = extract_inner_hash(blob).unwrap();
+ assert_eq!(store.get(&h).unwrap(), big);
+}
+
+// ─── helpers ──────────────────────────────────────────────────────
+
+/// Pull the hash out of a `<<ccr:HASH N_rows_offloaded>>` row marker.
+fn extract_hash_from_marker(s: &str) -> Option<String> {
+ let i = s.find("<<ccr:")?;
+ let after = &s[i + 6..];
+ let end = after.find(' ')?;
+ Some(after[..end].to_string())
+}
+
+/// Pull the hash out of a `<<ccr:HASH,KIND,SIZE>>` opaque marker.
+fn extract_inner_hash(s: &str) -> Option<String> {
+ let i = s.find("<<ccr:")?;
+ let after = &s[i + 6..];
+ let end = after.find(',')?;
+ Some(after[..end].to_string())
+}
diff --git a/crates/headroom-py/src/lib.rs b/crates/headroom-py/src/lib.rs
index 70197de8e..b5b0eb36a 100644
--- a/crates/headroom-py/src/lib.rs
+++ b/crates/headroom-py/src/lib.rs
@@ -15,6 +15,7 @@
use std::collections::BTreeMap;
+use headroom_core::transforms::smart_crusher::compaction::DocumentCompactor;
use headroom_core::transforms::smart_crusher::{
CrushResult as RustCrushResult, SmartCrusher as RustSmartCrusher,
SmartCrusherConfig as RustSmartCrusherConfig,
@@ -705,6 +706,30 @@ impl PySmartCrusher {
)
}
+ /// Run the document-level walker on `doc_json` (JSON string) and
+ /// return the compacted document as JSON.
+ ///
+ /// The walker recursively descends through objects, arrays, and
+ /// strings; tabular sub-arrays become rendered CSV+schema strings,
+ /// long opaque blobs become `<<ccr:HASH,KIND,SIZE>>` markers (with
+ /// originals stashed in this crusher's CCR store, so `ccr_get`
+ /// resolves them).
+ ///
+ /// Distinct from `crush_array_json`: this is the lossless walker
+ /// pass without per-array lossy crushing — useful when the caller
+ /// wants document-shape compaction (forms, configs, mixed records)
+ /// rather than statistical row drop.
+ fn compact_document_json(&self, doc_json: &str) -> String {
+ let parsed: serde_json::Value =
+ serde_json::from_str(doc_json).unwrap_or_else(|e| panic!("doc_json must be JSON: {e}"));
+ let mut dc = DocumentCompactor::new();
+ if let Some(store) = self.inner.ccr_store() {
+ dc = dc.with_ccr_store(store.clone());
+ }
+ let out = dc.compact(parsed);
+ serde_json::to_string(&out).expect("serialize compacted document")
+ }
+
/// Look up an original payload by CCR hash.
///
/// When the lossy path drops rows, it stashes the **full original**
diff --git a/headroom/transforms/smart_crusher.py b/headroom/transforms/smart_crusher.py
index 3dc7819c3..1d1d3abd4 100644
--- a/headroom/transforms/smart_crusher.py
+++ b/headroom/transforms/smart_crusher.py
@@ -214,6 +214,20 @@ class SmartCrusher(Transform):
result: dict[str, Any] = self._rust.crush_array_json(items_json, query, bias)
return result
+ def compact_document_json(self, doc_json: str) -> str:
+ """Run the document walker on ``doc_json`` and return compacted JSON.
+
+ Lossless walker pass over objects, arrays, and strings —
+ tabular sub-arrays become CSV+schema strings, long opaque
+ blobs become ``<<ccr:HASH,KIND,SIZE>>`` markers (originals
+ stashed in this crusher's CCR store, so ``ccr_get`` resolves them).
+
+ Use this when callers want pure document-shape compaction
+ without per-array lossy crushing.
+ """
+ result: str = self._rust.compact_document_json(doc_json)
+ return result
+
def ccr_get(self, hash_key: str) -> str | None:
"""Look up an original payload by CCR hash from the Rust store.
diff --git a/tests/parity/fixtures/smart_crusher/dict_array_100_sequential_b91d1fff5ba3.json b/tests/parity/fixtures/smart_crusher/dict_array_100_sequential_b91d1fff5ba3.json
index 7810da620..84899845e 100644
--- a/tests/parity/fixtures/smart_crusher/dict_array_100_sequential_b91d1fff5ba3.json
+++ b/tests/parity/fixtures/smart_crusher/dict_array_100_sequential_b91d1fff5ba3.json
@@ -24,7 +24,7 @@
"input_sha256": "b91d1fff5ba3ca82d797014b6f93c01e5e4ab7e2f3667c463a0015a2166617c3",
"label": "dict_array_100_sequential",
"output": {
- "compressed": "[{\"id\":0,\"level\":\"warn\",\"message\":\"seq 0\"},{\"id\":7,\"level\":\"warn\",\"message\":\"seq 7\"},{\"id\":14,\"level\":\"warn\",\"message\":\"seq 14\"},{\"id\":21,\"level\":\"warn\",\"message\":\"seq 21\"},{\"id\":28,\"level\":\"warn\",\"message\":\"seq 28\"},{\"id\":35,\"level\":\"warn\",\"message\":\"seq 35\"},{\"id\":42,\"level\":\"warn\",\"message\":\"seq 42\"},{\"id\":49,\"level\":\"warn\",\"message\":\"seq 49\"},{\"id\":56,\"level\":\"warn\",\"message\":\"seq 56\"},{\"id\":63,\"level\":\"warn\",\"message\":\"seq 63\"},{\"id\":70,\"level\":\"warn\",\"message\":\"seq 70\"},{\"id\":77,\"level\":\"warn\",\"message\":\"seq 77\"},{\"id\":84,\"level\":\"warn\",\"message\":\"seq 84\"},{\"id\":91,\"level\":\"warn\",\"message\":\"seq 91\"},{\"id\":98,\"level\":\"warn\",\"message\":\"seq 98\"}]",
+ "compressed": "[{\"id\":0,\"level\":\"warn\",\"message\":\"seq 0\"},{\"id\":7,\"level\":\"warn\",\"message\":\"seq 7\"},{\"id\":14,\"level\":\"warn\",\"message\":\"seq 14\"},{\"id\":21,\"level\":\"warn\",\"message\":\"seq 21\"},{\"id\":28,\"level\":\"warn\",\"message\":\"seq 28\"},{\"id\":35,\"level\":\"warn\",\"message\":\"seq 35\"},{\"id\":42,\"level\":\"warn\",\"message\":\"seq 42\"},{\"id\":49,\"level\":\"warn\",\"message\":\"seq 49\"},{\"id\":56,\"level\":\"warn\",\"message\":\"seq 56\"},{\"id\":63,\"level\":\"warn\",\"message\":\"seq 63\"},{\"id\":70,\"level\":\"warn\",\"message\":\"seq 70\"},{\"id\":77,\"level\":\"warn\",\"message\":\"seq 77\"},{\"id\":84,\"level\":\"warn\",\"message\":\"seq 84\"},{\"id\":91,\"level\":\"warn\",\"message\":\"seq 91\"},{\"id\":98,\"level\":\"warn\",\"message\":\"seq 98\"},{\"_ccr_dropped\":\"<<ccr:1d0dd94cf2cd 85_rows_offloaded>>\"}]",
"original": "[{\"id\": 0, \"level\": \"warn\", \"message\": \"seq 0\"}, {\"id\": 1, \"level\": \"info\", \"message\": \"seq 1\"}, {\"id\": 2, \"level\": \"info\", \"message\": \"seq 2\"}, {\"id\": 3, \"level\": \"info\", \"message\": \"seq 3\"}, {\"id\": 4, \"level\": \"info\", \"message\": \"seq 4\"}, {\"id\": 5, \"level\": \"info\", \"message\": \"seq 5\"}, {\"id\": 6, \"level\": \"info\", \"message\": \"seq 6\"}, {\"id\": 7, \"level\": \"warn\", \"message\": \"seq 7\"}, {\"id\": 8, \"level\": \"info\", \"message\": \"seq 8\"}, {\"id\": 9, \"level\": \"info\", \"message\": \"seq 9\"}, {\"id\": 10, \"level\": \"info\", \"message\": \"seq 10\"}, {\"id\": 11, \"level\": \"info\", \"message\": \"seq 11\"}, {\"id\": 12, \"level\": \"info\", \"message\": \"seq 12\"}, {\"id\": 13, \"level\": \"info\", \"message\": \"seq 13\"}, {\"id\": 14, \"level\": \"warn\", \"message\": \"seq 14\"}, {\"id\": 15, \"level\": \"info\", \"message\": \"seq 15\"}, {\"id\": 16, \"level\": \"info\", \"message\": \"seq 16\"}, {\"id\": 17, \"level\": \"info\", \"message\": \"seq 17\"}, {\"id\": 18, \"level\": \"info\", \"message\": \"seq 18\"}, {\"id\": 19, \"level\": \"info\", \"message\": \"seq 19\"}, {\"id\": 20, \"level\": \"info\", \"message\": \"seq 20\"}, {\"id\": 21, \"level\": \"warn\", \"message\": \"seq 21\"}, {\"id\": 22, \"level\": \"info\", \"message\": \"seq 22\"}, {\"id\": 23, \"level\": \"info\", \"message\": \"seq 23\"}, {\"id\": 24, \"level\": \"info\", \"message\": \"seq 24\"}, {\"id\": 25, \"level\": \"info\", \"message\": \"seq 25\"}, {\"id\": 26, \"level\": \"info\", \"message\": \"seq 26\"}, {\"id\": 27, \"level\": \"info\", \"message\": \"seq 27\"}, {\"id\": 28, \"level\": \"warn\", \"message\": \"seq 28\"}, {\"id\": 29, \"level\": \"info\", \"message\": \"seq 29\"}, {\"id\": 30, \"level\": \"info\", \"message\": \"seq 30\"}, {\"id\": 31, \"level\": \"info\", \"message\": \"seq 31\"}, {\"id\": 32, \"level\": \"info\", \"message\": \"seq 32\"}, {\"id\": 33, \"level\": \"info\", \"message\": \"seq 33\"}, {\"id\": 34, \"level\": \"info\", \"message\": \"seq 34\"}, {\"id\": 35, \"level\": \"warn\", \"message\": \"seq 35\"}, {\"id\": 36, \"level\": \"info\", \"message\": \"seq 36\"}, {\"id\": 37, \"level\": \"info\", \"message\": \"seq 37\"}, {\"id\": 38, \"level\": \"info\", \"message\": \"seq 38\"}, {\"id\": 39, \"level\": \"info\", \"message\": \"seq 39\"}, {\"id\": 40, \"level\": \"info\", \"message\": \"seq 40\"}, {\"id\": 41, \"level\": \"info\", \"message\": \"seq 41\"}, {\"id\": 42, \"level\": \"warn\", \"message\": \"seq 42\"}, {\"id\": 43, \"level\": \"info\", \"message\": \"seq 43\"}, {\"id\": 44, \"level\": \"info\", \"message\": \"seq 44\"}, {\"id\": 45, \"level\": \"info\", \"message\": \"seq 45\"}, {\"id\": 46, \"level\": \"info\", \"message\": \"seq 46\"}, {\"id\": 47, \"level\": \"info\", \"message\": \"seq 47\"}, {\"id\": 48, \"level\": \"info\", \"message\": \"seq 48\"}, {\"id\": 49, \"level\": \"warn\", \"message\": \"seq 49\"}, {\"id\": 50, \"level\": \"info\", \"message\": \"seq 50\"}, {\"id\": 51, \"level\": \"info\", \"message\": \"seq 51\"}, {\"id\": 52, \"level\": \"info\", \"message\": \"seq 52\"}, {\"id\": 53, \"level\": \"info\", \"message\": \"seq 53\"}, {\"id\": 54, \"level\": \"info\", \"message\": \"seq 54\"}, {\"id\": 55, \"level\": \"info\", \"message\": \"seq 55\"}, {\"id\": 56, \"level\": \"warn\", \"message\": \"seq 56\"}, {\"id\": 57, \"level\": \"info\", \"message\": \"seq 57\"}, {\"id\": 58, \"level\": \"info\", \"message\": \"seq 58\"}, {\"id\": 59, \"level\": \"info\", \"message\": \"seq 59\"}, {\"id\": 60, \"level\": \"info\", \"message\": \"seq 60\"}, {\"id\": 61, \"level\": \"info\", \"message\": \"seq 61\"}, {\"id\": 62, \"level\": \"info\", \"message\": \"seq 62\"}, {\"id\": 63, \"level\": \"warn\", \"message\": \"seq 63\"}, {\"id\": 64, \"level\": \"info\", \"message\": \"seq 64\"}, {\"id\": 65, \"level\": \"info\", \"message\": \"seq 65\"}, {\"id\": 66, \"level\": \"info\", \"message\": \"seq 66\"}, {\"id\": 67, \"level\": \"info\", \"message\": \"seq 67\"}, {\"id\": 68, \"level\": \"info\", \"message\": \"seq 68\"}, {\"id\": 69, \"level\": \"info\", \"message\": \"seq 69\"}, {\"id\": 70, \"level\": \"warn\", \"message\": \"seq 70\"}, {\"id\": 71, \"level\": \"info\", \"message\": \"seq 71\"}, {\"id\": 72, \"level\": \"info\", \"message\": \"seq 72\"}, {\"id\": 73, \"level\": \"info\", \"message\": \"seq 73\"}, {\"id\": 74, \"level\": \"info\", \"message\": \"seq 74\"}, {\"id\": 75, \"level\": \"info\", \"message\": \"seq 75\"}, {\"id\": 76, \"level\": \"info\", \"message\": \"seq 76\"}, {\"id\": 77, \"level\": \"warn\", \"message\": \"seq 77\"}, {\"id\": 78, \"level\": \"info\", \"message\": \"seq 78\"}, {\"id\": 79, \"level\": \"info\", \"message\": \"seq 79\"}, {\"id\": 80, \"level\": \"info\", \"message\": \"seq 80\"}, {\"id\": 81, \"level\": \"info\", \"message\": \"seq 81\"}, {\"id\": 82, \"level\": \"info\", \"message\": \"seq 82\"}, {\"id\": 83, \"level\": \"info\", \"message\": \"seq 83\"}, {\"id\": 84, \"level\": \"warn\", \"message\": \"seq 84\"}, {\"id\": 85, \"level\": \"info\", \"message\": \"seq 85\"}, {\"id\": 86, \"level\": \"info\", \"message\": \"seq 86\"}, {\"id\": 87, \"level\": \"info\", \"message\": \"seq 87\"}, {\"id\": 88, \"level\": \"info\", \"message\": \"seq 88\"}, {\"id\": 89, \"level\": \"info\", \"message\": \"seq 89\"}, {\"id\": 90, \"level\": \"info\", \"message\": \"seq 90\"}, {\"id\": 91, \"level\": \"warn\", \"message\": \"seq 91\"}, {\"id\": 92, \"level\": \"info\", \"message\": \"seq 92\"}, {\"id\": 93, \"level\": \"info\", \"message\": \"seq 93\"}, {\"id\": 94, \"level\": \"info\", \"message\": \"seq 94\"}, {\"id\": 95, \"level\": \"info\", \"message\": \"seq 95\"}, {\"id\": 96, \"level\": \"info\", \"message\": \"seq 96\"}, {\"id\": 97, \"level\": \"info\", \"message\": \"seq 97\"}, {\"id\": 98, \"level\": \"warn\", \"message\": \"seq 98\"}, {\"id\": 99, \"level\": \"info\", \"message\": \"seq 99\"}]",
"strategy": "smart_sample(100->15)",
"was_modified": true
diff --git a/tests/parity/fixtures/smart_crusher/dict_array_30_b0ba1d26fd03.json b/tests/parity/fixtures/smart_crusher/dict_array_30_b0ba1d26fd03.json
index 8da36040d..77291ad48 100644
--- a/tests/parity/fixtures/smart_crusher/dict_array_30_b0ba1d26fd03.json
+++ b/tests/parity/fixtures/smart_crusher/dict_array_30_b0ba1d26fd03.json
@@ -24,7 +24,7 @@
"input_sha256": "b0ba1d26fd03d9b5010ffd8d4c6185e0c5baf03ab783316f3ad15830d1bba779",
"label": "dict_array_30",
"output": {
- "compressed": "[{\"id\":0,\"status\":\"error\",\"msg\":\"line 0\"},{\"id\":1,\"status\":\"ok\",\"msg\":\"line 1\"},{\"id\":2,\"status\":\"ok\",\"msg\":\"line 2\"},{\"id\":4,\"status\":\"ok\",\"msg\":\"line 4\"},{\"id\":5,\"status\":\"error\",\"msg\":\"line 5\"},{\"id\":7,\"status\":\"ok\",\"msg\":\"line 7\"},{\"id\":9,\"status\":\"ok\",\"msg\":\"line 9\"},{\"id\":10,\"status\":\"error\",\"msg\":\"line 10\"},{\"id\":12,\"status\":\"ok\",\"msg\":\"line 12\"},{\"id\":14,\"status\":\"ok\",\"msg\":\"line 14\"},{\"id\":15,\"status\":\"error\",\"msg\":\"line 15\"},{\"id\":17,\"status\":\"ok\",\"msg\":\"line 17\"},{\"id\":20,\"status\":\"error\",\"msg\":\"line 20\"},{\"id\":25,\"status\":\"error\",\"msg\":\"line 25\"},{\"id\":29,\"status\":\"ok\",\"msg\":\"line 29\"}]",
+ "compressed": "[{\"id\":0,\"status\":\"error\",\"msg\":\"line 0\"},{\"id\":1,\"status\":\"ok\",\"msg\":\"line 1\"},{\"id\":2,\"status\":\"ok\",\"msg\":\"line 2\"},{\"id\":4,\"status\":\"ok\",\"msg\":\"line 4\"},{\"id\":5,\"status\":\"error\",\"msg\":\"line 5\"},{\"id\":7,\"status\":\"ok\",\"msg\":\"line 7\"},{\"id\":9,\"status\":\"ok\",\"msg\":\"line 9\"},{\"id\":10,\"status\":\"error\",\"msg\":\"line 10\"},{\"id\":12,\"status\":\"ok\",\"msg\":\"line 12\"},{\"id\":14,\"status\":\"ok\",\"msg\":\"line 14\"},{\"id\":15,\"status\":\"error\",\"msg\":\"line 15\"},{\"id\":17,\"status\":\"ok\",\"msg\":\"line 17\"},{\"id\":20,\"status\":\"error\",\"msg\":\"line 20\"},{\"id\":25,\"status\":\"error\",\"msg\":\"line 25\"},{\"id\":29,\"status\":\"ok\",\"msg\":\"line 29\"},{\"_ccr_dropped\":\"<<ccr:89f81e97033e 15_rows_offloaded>>\"}]",
"original": "[{\"id\": 0, \"status\": \"error\", \"msg\": \"line 0\"}, {\"id\": 1, \"status\": \"ok\", \"msg\": \"line 1\"}, {\"id\": 2, \"status\": \"ok\", \"msg\": \"line 2\"}, {\"id\": 3, \"status\": \"ok\", \"msg\": \"line 3\"}, {\"id\": 4, \"status\": \"ok\", \"msg\": \"line 4\"}, {\"id\": 5, \"status\": \"error\", \"msg\": \"line 5\"}, {\"id\": 6, \"status\": \"ok\", \"msg\": \"line 6\"}, {\"id\": 7, \"status\": \"ok\", \"msg\": \"line 7\"}, {\"id\": 8, \"status\": \"ok\", \"msg\": \"line 8\"}, {\"id\": 9, \"status\": \"ok\", \"msg\": \"line 9\"}, {\"id\": 10, \"status\": \"error\", \"msg\": \"line 10\"}, {\"id\": 11, \"status\": \"ok\", \"msg\": \"line 11\"}, {\"id\": 12, \"status\": \"ok\", \"msg\": \"line 12\"}, {\"id\": 13, \"status\": \"ok\", \"msg\": \"line 13\"}, {\"id\": 14, \"status\": \"ok\", \"msg\": \"line 14\"}, {\"id\": 15, \"status\": \"error\", \"msg\": \"line 15\"}, {\"id\": 16, \"status\": \"ok\", \"msg\": \"line 16\"}, {\"id\": 17, \"status\": \"ok\", \"msg\": \"line 17\"}, {\"id\": 18, \"status\": \"ok\", \"msg\": \"line 18\"}, {\"id\": 19, \"status\": \"ok\", \"msg\": \"line 19\"}, {\"id\": 20, \"status\": \"error\", \"msg\": \"line 20\"}, {\"id\": 21, \"status\": \"ok\", \"msg\": \"line 21\"}, {\"id\": 22, \"status\": \"ok\", \"msg\": \"line 22\"}, {\"id\": 23, \"status\": \"ok\", \"msg\": \"line 23\"}, {\"id\": 24, \"status\": \"ok\", \"msg\": \"line 24\"}, {\"id\": 25, \"status\": \"error\", \"msg\": \"line 25\"}, {\"id\": 26, \"status\": \"ok\", \"msg\": \"line 26\"}, {\"id\": 27, \"status\": \"ok\", \"msg\": \"line 27\"}, {\"id\": 28, \"status\": \"ok\", \"msg\": \"line 28\"}, {\"id\": 29, \"status\": \"ok\", \"msg\": \"line 29\"}]",
"strategy": "smart_sample(30->15)",
"was_modified": true
diff --git a/tests/parity/fixtures/smart_crusher/dict_array_30_bias_high_dc3b36e60560.json b/tests/parity/fixtures/smart_crusher/dict_array_30_bias_high_dc3b36e60560.json
index 9dbeb71d7..50241a4e3 100644
--- a/tests/parity/fixtures/smart_crusher/dict_array_30_bias_high_dc3b36e60560.json
+++ b/tests/parity/fixtures/smart_crusher/dict_array_30_bias_high_dc3b36e60560.json
@@ -24,7 +24,7 @@
"input_sha256": "dc3b36e605600ba3d7b91fae6a05d2fb3c849de875c455530d61ed40bbb2c956",
"label": "dict_array_30_bias_high",
"output": {
- "compressed": "[{\"id\":0,\"status\":\"error\",\"msg\":\"line 0\"},{\"id\":1,\"status\":\"ok\",\"msg\":\"line 1\"},{\"id\":2,\"status\":\"ok\",\"msg\":\"line 2\"},{\"id\":4,\"status\":\"ok\",\"msg\":\"line 4\"},{\"id\":5,\"status\":\"error\",\"msg\":\"line 5\"},{\"id\":7,\"status\":\"ok\",\"msg\":\"line 7\"},{\"id\":9,\"status\":\"ok\",\"msg\":\"line 9\"},{\"id\":10,\"status\":\"error\",\"msg\":\"line 10\"},{\"id\":12,\"status\":\"ok\",\"msg\":\"line 12\"},{\"id\":14,\"status\":\"ok\",\"msg\":\"line 14\"},{\"id\":15,\"status\":\"error\",\"msg\":\"line 15\"},{\"id\":17,\"status\":\"ok\",\"msg\":\"line 17\"},{\"id\":20,\"status\":\"error\",\"msg\":\"line 20\"},{\"id\":25,\"status\":\"error\",\"msg\":\"line 25\"},{\"id\":29,\"status\":\"ok\",\"msg\":\"line 29\"}]",
+ "compressed": "[{\"id\":0,\"status\":\"error\",\"msg\":\"line 0\"},{\"id\":1,\"status\":\"ok\",\"msg\":\"line 1\"},{\"id\":2,\"status\":\"ok\",\"msg\":\"line 2\"},{\"id\":4,\"status\":\"ok\",\"msg\":\"line 4\"},{\"id\":5,\"status\":\"error\",\"msg\":\"line 5\"},{\"id\":7,\"status\":\"ok\",\"msg\":\"line 7\"},{\"id\":9,\"status\":\"ok\",\"msg\":\"line 9\"},{\"id\":10,\"status\":\"error\",\"msg\":\"line 10\"},{\"id\":12,\"status\":\"ok\",\"msg\":\"line 12\"},{\"id\":14,\"status\":\"ok\",\"msg\":\"line 14\"},{\"id\":15,\"status\":\"error\",\"msg\":\"line 15\"},{\"id\":17,\"status\":\"ok\",\"msg\":\"line 17\"},{\"id\":20,\"status\":\"error\",\"msg\":\"line 20\"},{\"id\":25,\"status\":\"error\",\"msg\":\"line 25\"},{\"id\":29,\"status\":\"ok\",\"msg\":\"line 29\"},{\"_ccr_dropped\":\"<<ccr:89f81e97033e 15_rows_offloaded>>\"}]",
"original": "[{\"id\": 0, \"status\": \"error\", \"msg\": \"line 0\"}, {\"id\": 1, \"status\": \"ok\", \"msg\": \"line 1\"}, {\"id\": 2, \"status\": \"ok\", \"msg\": \"line 2\"}, {\"id\": 3, \"status\": \"ok\", \"msg\": \"line 3\"}, {\"id\": 4, \"status\": \"ok\", \"msg\": \"line 4\"}, {\"id\": 5, \"status\": \"error\", \"msg\": \"line 5\"}, {\"id\": 6, \"status\": \"ok\", \"msg\": \"line 6\"}, {\"id\": 7, \"status\": \"ok\", \"msg\": \"line 7\"}, {\"id\": 8, \"status\": \"ok\", \"msg\": \"line 8\"}, {\"id\": 9, \"status\": \"ok\", \"msg\": \"line 9\"}, {\"id\": 10, \"status\": \"error\", \"msg\": \"line 10\"}, {\"id\": 11, \"status\": \"ok\", \"msg\": \"line 11\"}, {\"id\": 12, \"status\": \"ok\", \"msg\": \"line 12\"}, {\"id\": 13, \"status\": \"ok\", \"msg\": \"line 13\"}, {\"id\": 14, \"status\": \"ok\", \"msg\": \"line 14\"}, {\"id\": 15, \"status\": \"error\", \"msg\": \"line 15\"}, {\"id\": 16, \"status\": \"ok\", \"msg\": \"line 16\"}, {\"id\": 17, \"status\": \"ok\", \"msg\": \"line 17\"}, {\"id\": 18, \"status\": \"ok\", \"msg\": \"line 18\"}, {\"id\": 19, \"status\": \"ok\", \"msg\": \"line 19\"}, {\"id\": 20, \"status\": \"error\", \"msg\": \"line 20\"}, {\"id\": 21, \"status\": \"ok\", \"msg\": \"line 21\"}, {\"id\": 22, \"status\": \"ok\", \"msg\": \"line 22\"}, {\"id\": 23, \"status\": \"ok\", \"msg\": \"line 23\"}, {\"id\": 24, \"status\": \"ok\", \"msg\": \"line 24\"}, {\"id\": 25, \"status\": \"error\", \"msg\": \"line 25\"}, {\"id\": 26, \"status\": \"ok\", \"msg\": \"line 26\"}, {\"id\": 27, \"status\": \"ok\", \"msg\": \"line 27\"}, {\"id\": 28, \"status\": \"ok\", \"msg\": \"line 28\"}, {\"id\": 29, \"status\": \"ok\", \"msg\": \"line 29\"}]",
"strategy": "smart_sample(30->15)",
"was_modified": true
diff --git a/tests/parity/fixtures/smart_crusher/dict_array_30_bias_low_a850512901bd.json b/tests/parity/fixtures/smart_crusher/dict_array_30_bias_low_a850512901bd.json
index 091b94ec6..0e5022bfa 100644
--- a/tests/parity/fixtures/smart_crusher/dict_array_30_bias_low_a850512901bd.json
+++ b/tests/parity/fixtures/smart_crusher/dict_array_30_bias_low_a850512901bd.json
@@ -24,7 +24,7 @@
"input_sha256": "a850512901bdb42373b58e753e7a9ed2ae20fab671f686f2fd7a1c670889eb93",
"label": "dict_array_30_bias_low",
"output": {
- "compressed": "[{\"id\":0,\"status\":\"error\",\"msg\":\"line 0\"},{\"id\":1,\"status\":\"ok\",\"msg\":\"line 1\"},{\"id\":2,\"status\":\"ok\",\"msg\":\"line 2\"},{\"id\":4,\"status\":\"ok\",\"msg\":\"line 4\"},{\"id\":5,\"status\":\"error\",\"msg\":\"line 5\"},{\"id\":7,\"status\":\"ok\",\"msg\":\"line 7\"},{\"id\":9,\"status\":\"ok\",\"msg\":\"line 9\"},{\"id\":10,\"status\":\"error\",\"msg\":\"line 10\"},{\"id\":12,\"status\":\"ok\",\"msg\":\"line 12\"},{\"id\":14,\"status\":\"ok\",\"msg\":\"line 14\"},{\"id\":15,\"status\":\"error\",\"msg\":\"line 15\"},{\"id\":17,\"status\":\"ok\",\"msg\":\"line 17\"},{\"id\":20,\"status\":\"error\",\"msg\":\"line 20\"},{\"id\":25,\"status\":\"error\",\"msg\":\"line 25\"},{\"id\":29,\"status\":\"ok\",\"msg\":\"line 29\"}]",
+ "compressed": "[{\"id\":0,\"status\":\"error\",\"msg\":\"line 0\"},{\"id\":1,\"status\":\"ok\",\"msg\":\"line 1\"},{\"id\":2,\"status\":\"ok\",\"msg\":\"line 2\"},{\"id\":4,\"status\":\"ok\",\"msg\":\"line 4\"},{\"id\":5,\"status\":\"error\",\"msg\":\"line 5\"},{\"id\":7,\"status\":\"ok\",\"msg\":\"line 7\"},{\"id\":9,\"status\":\"ok\",\"msg\":\"line 9\"},{\"id\":10,\"status\":\"error\",\"msg\":\"line 10\"},{\"id\":12,\"status\":\"ok\",\"msg\":\"line 12\"},{\"id\":14,\"status\":\"ok\",\"msg\":\"line 14\"},{\"id\":15,\"status\":\"error\",\"msg\":\"line 15\"},{\"id\":17,\"status\":\"ok\",\"msg\":\"line 17\"},{\"id\":20,\"status\":\"error\",\"msg\":\"line 20\"},{\"id\":25,\"status\":\"error\",\"msg\":\"line 25\"},{\"id\":29,\"status\":\"ok\",\"msg\":\"line 29\"},{\"_ccr_dropped\":\"<<ccr:89f81e97033e 15_rows_offloaded>>\"}]",
"original": "[{\"id\": 0, \"status\": \"error\", \"msg\": \"line 0\"}, {\"id\": 1, \"status\": \"ok\", \"msg\": \"line 1\"}, {\"id\": 2, \"status\": \"ok\", \"msg\": \"line 2\"}, {\"id\": 3, \"status\": \"ok\", \"msg\": \"line 3\"}, {\"id\": 4, \"status\": \"ok\", \"msg\": \"line 4\"}, {\"id\": 5, \"status\": \"error\", \"msg\": \"line 5\"}, {\"id\": 6, \"status\": \"ok\", \"msg\": \"line 6\"}, {\"id\": 7, \"status\": \"ok\", \"msg\": \"line 7\"}, {\"id\": 8, \"status\": \"ok\", \"msg\": \"line 8\"}, {\"id\": 9, \"status\": \"ok\", \"msg\": \"line 9\"}, {\"id\": 10, \"status\": \"error\", \"msg\": \"line 10\"}, {\"id\": 11, \"status\": \"ok\", \"msg\": \"line 11\"}, {\"id\": 12, \"status\": \"ok\", \"msg\": \"line 12\"}, {\"id\": 13, \"status\": \"ok\", \"msg\": \"line 13\"}, {\"id\": 14, \"status\": \"ok\", \"msg\": \"line 14\"}, {\"id\": 15, \"status\": \"error\", \"msg\": \"line 15\"}, {\"id\": 16, \"status\": \"ok\", \"msg\": \"line 16\"}, {\"id\": 17, \"status\": \"ok\", \"msg\": \"line 17\"}, {\"id\": 18, \"status\": \"ok\", \"msg\": \"line 18\"}, {\"id\": 19, \"status\": \"ok\", \"msg\": \"line 19\"}, {\"id\": 20, \"status\": \"error\", \"msg\": \"line 20\"}, {\"id\": 21, \"status\": \"ok\", \"msg\": \"line 21\"}, {\"id\": 22, \"status\": \"ok\", \"msg\": \"line 22\"}, {\"id\": 23, \"status\": \"ok\", \"msg\": \"line 23\"}, {\"id\": 24, \"status\": \"ok\", \"msg\": \"line 24\"}, {\"id\": 25, \"status\": \"error\", \"msg\": \"line 25\"}, {\"id\": 26, \"status\": \"ok\", \"msg\": \"line 26\"}, {\"id\": 27, \"status\": \"ok\", \"msg\": \"line 27\"}, {\"id\": 28, \"status\": \"ok\", \"msg\": \"line 28\"}, {\"id\": 29, \"status\": \"ok\", \"msg\": \"line 29\"}]",
"strategy": "smart_sample(30->15)",
"was_modified": true
diff --git a/tests/parity/fixtures/smart_crusher/duplicate_dicts_40_40a0670dec17.json b/tests/parity/fixtures/smart_crusher/duplicate_dicts_40_40a0670dec17.json
index e1d0e4c91..e8a2a55a5 100644
--- a/tests/parity/fixtures/smart_crusher/duplicate_dicts_40_40a0670dec17.json
+++ b/tests/parity/fixtures/smart_crusher/duplicate_dicts_40_40a0670dec17.json
@@ -24,7 +24,7 @@
"input_sha256": "40a0670dec17ae0f765b640ffea2a78c2f2526c03877dcabf0b6108b608ada8a",
"label": "duplicate_dicts_40",
"output": {
- "compressed": "[{\"event\":\"heartbeat\",\"ok\":true}]",
+ "compressed": "[{\"event\":\"heartbeat\",\"ok\":true},{\"_ccr_dropped\":\"<<ccr:6155f8b8951e 39_rows_offloaded>>\"}]",
"original": "[{\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}]",
"strategy": "smart_sample(40->1)",
"was_modified": true
diff --git a/tests/test_transforms/test_smart_crusher_ccr_roundtrip.py b/tests/test_transforms/test_smart_crusher_ccr_roundtrip.py
index 69018fe04..6cebf15e9 100644
--- a/tests/test_transforms/test_smart_crusher_ccr_roundtrip.py
+++ b/tests/test_transforms/test_smart_crusher_ccr_roundtrip.py
@@ -239,6 +239,87 @@ def test_kept_subset_is_subset_of_original() -> None:
assert retrieved == original
+def test_marker_visible_in_crush_output_native() -> None:
+ """PR8 cornerstone: the public crush() output now carries the
+ `<<ccr:HASH ...>>` marker so the LLM sees the retrieval pointer."""
+ from headroom._core import SmartCrusher, SmartCrusherConfig
+
+ crusher = SmartCrusher(SmartCrusherConfig(lossless_min_savings_ratio=0.99))
+ items = [{"id": i, "status": "ok"} for i in range(50)]
+ raw = json.dumps(items)
+
+ result = crusher.crush(raw, "", 1.0)
+ assert "<<ccr:" in result.compressed, f"expected marker in output: {result.compressed[:200]!r}"
+ assert "rows_offloaded" in result.compressed
+
+ # The marker hash resolves in the store.
+ import re
+
+ m = re.search(r"<<ccr:([a-f0-9]+) ", result.compressed)
+ assert m is not None
+ hash_key = m.group(1)
+ assert crusher.ccr_get(hash_key) is not None
+
+
+def test_opaque_blob_in_object_emits_marker_and_stores_native() -> None:
+ """A long base64-ish blob in a field becomes a CCR marker AND the
+ original gets stashed."""
+ from headroom._core import SmartCrusher
+
+ crusher = SmartCrusher()
+ big = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=" * 8
+ raw = json.dumps({"id": 1, "blob": big})
+
+ result = crusher.crush(raw, "", 1.0)
+ parsed = json.loads(result.compressed)
+ blob_out = parsed["blob"]
+ assert blob_out.startswith("<<ccr:") and ",base64," in blob_out
+
+ # The store grew, hash resolves, original byte-equal.
+ import re
+
+ m = re.search(r"<<ccr:([a-f0-9]+),", blob_out)
+ assert m is not None
+ retrieved = crusher.ccr_get(m.group(1))
+ assert retrieved == big
+
+
+def test_compact_document_json_via_pyo3() -> None:
+ """The walker is reachable from Python and writes to the same store."""
+ from headroom._core import SmartCrusher
+
+ crusher = SmartCrusher()
+ starting = crusher.ccr_len()
+ big = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=" * 8
+ doc = {"id": 1, "blob": big}
+
+ out = crusher.compact_document_json(json.dumps(doc))
+ parsed = json.loads(out)
+ assert parsed["id"] == 1
+ assert parsed["blob"].startswith("<<ccr:")
+ assert crusher.ccr_len() == starting + 1
+
+ import re
+
+ m = re.search(r"<<ccr:([a-f0-9]+),", parsed["blob"])
+ assert crusher.ccr_get(m.group(1)) == big
+
+
+def test_compact_document_via_shim() -> None:
+ """Same path via the Python shim."""
+ from headroom.config import SmartCrusherConfig as PyConfig
+ from headroom.transforms.smart_crusher import SmartCrusher
+
+ crusher = SmartCrusher(PyConfig())
+ items = [{"id": i, "status": "ok", "tag": "alpha"} for i in range(30)]
+ out = crusher.compact_document_json(json.dumps({"events": items}))
+ parsed = json.loads(out)
+ # Tabular sub-array compacted to a string.
+ assert isinstance(parsed["events"], str), (
+ f"expected string, got {type(parsed['events']).__name__}"
+ )
+
+
def test_distinct_payloads_have_distinct_hashes_and_separate_storage() -> None:
"""Two different payloads → two different hashes → both
independently retrievable. Pins the per-payload isolation."""