Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
chore(rust): SmartCrusher CCR marker injection + walker unification
chopratejasda7716a95af0
diff --git a/crates/headroom-core/src/transforms/smart_crusher/compaction/mod.rs b/crates/headroom-core/src/transforms/smart_crusher/compaction/mod.rsindex 5b94dd434..cecfe60d7 100644--- a/crates/headroom-core/src/transforms/smart_crusher/compaction/mod.rs+++ b/crates/headroom-core/src/transforms/smart_crusher/compaction/mod.rs@@ -33,7 +33,9 @@ pub use classifier::{classify_cell, CellClass, ClassifyConfig};pub use compactor::{compact, CompactConfig};pub use formatter::{CsvSchemaFormatter, Formatter, JsonFormatter};pub use ir::{Bucket, CellValue, Compaction, FieldSpec, OpaqueKind, Row, Schema};-pub use walker::{compact_document, DocumentCompactor};+pub use walker::{+ compact_document, emit_opaque_ccr_marker, try_parse_json_container, DocumentCompactor,+};/// Composed compaction stage: a config + formatter pair.///diff --git a/crates/headroom-core/src/transforms/smart_crusher/compaction/walker.rs b/crates/headroom-core/src/transforms/smart_crusher/compaction/walker.rsindex 59e8631c1..1d6672a2e 100644--- a/crates/headroom-core/src/transforms/smart_crusher/compaction/walker.rs+++ b/crates/headroom-core/src/transforms/smart_crusher/compaction/walker.rs@@ -29,12 +29,15 @@//! BEFORE running TabularCompactor on the array, so inner sub-tables//! become strings first and the outer table sees them as cells.+use std::sync::Arc;+use serde_json::{Map, Value};use super::classifier::{classify_cell, CellClass};use super::compactor::{compact, CompactConfig};use super::formatter::{CsvSchemaFormatter, Formatter};use super::ir::OpaqueKind;+use crate::ccr::CcrStore;use sha2::{Digest, Sha256};@@ -50,6 +53,10 @@ use sha2::{Digest, Sha256};pub struct DocumentCompactor {pub config: CompactConfig,pub formatter: Box<dyn Formatter>,+ /// Optional CCR store. When set, opaque-string CCR markers also+ /// stash the original blob keyed by the marker hash, mirroring the+ /// row-drop CCR contract from `SmartCrusher::crush_array`.+ pub ccr_store: Option<Arc<dyn CcrStore>>,}impl Default for DocumentCompactor {@@ -57,6 +64,7 @@ impl Default for DocumentCompactor {Self {config: CompactConfig::default(),formatter: Box::new(CsvSchemaFormatter::new()),+ ccr_store: None,}}}@@ -76,6 +84,11 @@ impl DocumentCompactor {self}+ pub fn with_ccr_store(mut self, store: Arc<dyn CcrStore>) -> Self {+ self.ccr_store = Some(store);+ self+ }+/// Walk and compact. Returns a JSON value with the same shape but/// with compactable spots replaced by rendered strings.pub fn compact(&self, doc: Value) -> Value {@@ -124,10 +137,11 @@ fn walk_string(s: String, ctx: &DocumentCompactor) -> Value {};}- // Long opaque blob: substitute with CCR marker.+ // Long opaque blob: substitute with CCR marker (and stash the+ // original in the store if one is configured, so retrieval works).if let CellClass::Opaque(kind) = classify_cell(&Value::String(s.clone()), &ctx.config.classify){- return Value::String(format_ccr_marker(s.as_bytes(), &kind));+ return Value::String(emit_opaque_ccr_marker(&s, &kind, ctx.ccr_store.as_ref()));}Value::String(s)@@ -136,7 +150,7 @@ fn walk_string(s: String, ctx: &DocumentCompactor) -> Value {/// Parse a string as JSON IF it looks like a container (starts with `{`/// or `[`) AND parses cleanly to Object/Array. Returns None otherwise —/// we don't recurse on bare scalars even if they parse.-fn try_parse_json_container(s: &str) -> Option<Value> {+pub fn try_parse_json_container(s: &str) -> Option<Value> {let trimmed = s.trim_start();if !matches!(trimmed.chars().next(), Some('{') | Some('[')) {return None;@@ -146,22 +160,37 @@ fn try_parse_json_container(s: &str) -> Option<Value> {.filter(|v| matches!(v, Value::Object(_) | Value::Array(_)))}-fn format_ccr_marker(bytes: &[u8], kind: &OpaqueKind) -> String {+/// Emit an opaque-blob CCR marker AND (optionally) stash the original+/// in the store so retrieval works. The hash is computed identically+/// regardless of store presence — same input → same marker — so the+/// runtime contract is stable across configurations.+///+/// Marker format: `<<ccr:HASH,KIND,SIZE>>` where HASH is the 12-char+/// SHA-256 hex prefix of the payload bytes, KIND is `base64` / `string`+/// / `html` / custom, SIZE is humanized (`123B`, `4.5KB`, `1.2MB`).+pub fn emit_opaque_ccr_marker(+ payload: &str,+ kind: &OpaqueKind,+ store: Option<&Arc<dyn CcrStore>>,+) -> String {let mut h = Sha256::new();- h.update(bytes);+ h.update(payload.as_bytes());let hash: String = h.finalize().iter().take(6).map(|b| format!("{b:02x}")).collect();+ if let Some(s) = store {+ s.put(&hash, payload);+ }let kind_str = match kind {OpaqueKind::Base64Blob => "base64",OpaqueKind::LongString => "string",OpaqueKind::HtmlChunk => "html",OpaqueKind::Other(s) => s.as_str(),};- format!("<<ccr:{},{},{}>>", hash, kind_str, humanize(bytes.len()))+ format!("<<ccr:{},{},{}>>", hash, kind_str, humanize(payload.len()))}fn humanize(n: usize) -> String {diff --git a/crates/headroom-core/src/transforms/smart_crusher/crusher.rs b/crates/headroom-core/src/transforms/smart_crusher/crusher.rsindex cf9739442..7b4a99654 100644--- a/crates/headroom-core/src/transforms/smart_crusher/crusher.rs+++ b/crates/headroom-core/src/transforms/smart_crusher/crusher.rs@@ -40,9 +40,10 @@ use serde_json::Value;use super::analyzer::SmartAnalyzer;use super::builder::SmartCrusherBuilder;use super::classifier::{classify_array, ArrayType};-use super::compaction::classifier::{classify_cell, CellClass, ClassifyConfig};-use super::compaction::ir::OpaqueKind;-use super::compaction::{Compaction, CompactionStage};+use super::compaction::{+ classify_cell, emit_opaque_ccr_marker, try_parse_json_container, CellClass, ClassifyConfig,+ Compaction, CompactionStage,+};use super::config::SmartCrusherConfig;use super::crushers::{compute_k_split, crush_number_array, crush_object, crush_string_array};use super::planning::SmartCrusherPlanner;@@ -390,7 +391,31 @@ impl SmartCrusher {n,result.items.len()));- return (Value::Array(result.items), info_parts.join(","));+ // Lossy path with rows dropped → append a+ // CCR-Dropped sentinel object as the last+ // element of the kept-items array. This is+ // the **only** place the LLM sees the+ // `<<ccr:HASH ...>>` pointer in the prompt.+ // Without this, the store has the data but+ // no model can ever ask for it.+ //+ // Sentinel shape: `{"_ccr_dropped":+ // "<<ccr:HASH N_rows_offloaded>>"}` —+ // preserves "array-of-objects" shape so+ // downstream consumers iterating with+ // `x.get(...)` keep working; the well-known+ // `_ccr_dropped` key signals metadata+ // unambiguously.+ let mut items = result.items;+ if !result.dropped_summary.is_empty() {+ let mut sentinel = serde_json::Map::new();+ sentinel.insert(+ "_ccr_dropped".to_string(),+ Value::String(result.dropped_summary),+ );+ items.push(Value::Object(sentinel));+ }+ return (Value::Array(items), info_parts.join(","));}ArrayType::StringArray => {let strs: Vec<&str> = arr.iter().filter_map(|v| v.as_str()).collect();@@ -451,11 +476,10 @@ impl SmartCrusher {(Value::Object(processed), info_parts.join(","))}- // Strings: walker-equivalent handling. Parse stringified-JSON- // containers and recurse; CCR-substitute long opaque blobs.- // (Stage 3c.2 PR5: brings DocumentCompactor's String semantics- // into the public `crush()` path so tool outputs that wrap- // JSON in strings, or contain opaque blobs, get processed.)+ // Strings: walker-equivalent handling. Delegates to+ // `process_string` which parses stringified-JSON containers+ // (recursing through `process_value`) and CCR-substitutes+ // opaque blobs (with store-write so retrieval works).Value::String(s) => self.process_string(s, depth, query_context, bias),// Other scalars — passthrough._ => (value.clone(), String::new()),@@ -486,7 +510,7 @@ impl SmartCrusher {bias: f64,) -> (Value, String) {// 1. Stringified-JSON: parse, recurse, re-render.- if let Some(parsed) = try_parse_json_container_str(s) {+ if let Some(parsed) = try_parse_json_container(s) {let (processed, sub_info) = self.process_value(&parsed, depth + 1, query_context, bias);// If recursion produced something different, re-emit.// Special case: if the recursion returned a `Value::String`@@ -509,14 +533,14 @@ impl SmartCrusher {}}- // 2. Opaque blob: substitute with CCR marker.+ // 2. Opaque blob: substitute with CCR marker AND stash the+ // original in the store (PR8) so retrieval works. Hash + format+ // identical to walker.rs via the shared helper — zero drift.let cfg = ClassifyConfig::default();if let CellClass::Opaque(kind) = classify_cell(&Value::String(s.to_string()), &cfg) {- let marker = ccr_marker_for_string(s.as_bytes(), &kind);- return (- Value::String(marker),- format!("string_ccr:{}", opaque_kind_label(&kind)),- );+ let marker = emit_opaque_ccr_marker(s, &kind, self.ccr_store.as_ref());+ let kind_label = opaque_kind_label(&kind);+ return (Value::String(marker), format!("string_ccr:{kind_label}"));}// 3. Plain string — passthrough.@@ -890,44 +914,19 @@ fn hash_array_for_ccr(items: &[Value]) -> String {.collect()}-// ─── PR5 walker-integration helpers (string handling) ───────────────────────--/// Cheap parse-as-JSON-container check. Returns `Some(parsed)` only-/// when `s` parses to an object or array — not for bare scalars-/// (numbers, bools, nulls) that happen to be valid JSON literals.-fn try_parse_json_container_str(s: &str) -> Option<Value> {- let trimmed = s.trim_start();- if !matches!(trimmed.chars().next(), Some('{') | Some('[')) {- return None;- }- serde_json::from_str::<Value>(s)- .ok()- .filter(|v| matches!(v, Value::Object(_) | Value::Array(_)))-}--/// Format a CCR retrieval marker for an opaque string. Format must-/// match `compaction::walker::format_ccr_marker` so downstream-/// consumers can pattern-match markers regardless of which path-/// emitted them.-fn ccr_marker_for_string(bytes: &[u8], kind: &OpaqueKind) -> String {- use sha2::{Digest, Sha256};- let mut h = Sha256::new();- h.update(bytes);- let hash: String = h- .finalize()- .iter()- .take(6)- .map(|b| format!("{b:02x}"))- .collect();- format!(- "<<ccr:{},{},{}>>",- hash,- opaque_kind_label(kind),- humanize_bytes(bytes.len())- )-}--fn opaque_kind_label(kind: &OpaqueKind) -> &str {+// ─── PR5 walker-integration helpers (string handling) ──────────────────────+//+// Parse-as-JSON-container, marker formatting, and humanize-bytes used to+// live here as locals. PR8 extracted them into `compaction::walker` so+// `walker.rs` and `process_value` share one canonical implementation —+// killing the drift risk where the two paths could format markers+// differently. `process_string` now calls `try_parse_json_container` and+// `emit_opaque_ccr_marker` directly. Only `opaque_kind_label` survives+// here because `process_string`'s `string_ccr:<kind>` strategy-info+// label is local to this module's debug-string convention.++fn opaque_kind_label(kind: &super::compaction::OpaqueKind) -> &str {+ use super::compaction::OpaqueKind;match kind {OpaqueKind::Base64Blob => "base64",OpaqueKind::LongString => "string",@@ -936,17 +935,6 @@ fn opaque_kind_label(kind: &OpaqueKind) -> &str {}}-fn humanize_bytes(n: usize) -> String {- if n < 1024 {- return format!("{n}B");- }- let kb = n as f64 / 1024.0;- if kb < 1024.0 {- return format!("{kb:.1}KB");- }- format!("{:.1}MB", kb / 1024.0)-}-#[cfg(test)]mod tests {use super::*;@@ -1423,11 +1411,11 @@ mod tests {#[test]fn process_string_helper_parses_only_containers() {- assert!(try_parse_json_container_str("{\"a\":1}").is_some());- assert!(try_parse_json_container_str("[1,2,3]").is_some());- assert!(try_parse_json_container_str("123").is_none()); // bare scalar- assert!(try_parse_json_container_str("\"hello\"").is_none()); // bare string- assert!(try_parse_json_container_str("not json").is_none());- assert!(try_parse_json_container_str("{malformed").is_none());+ assert!(try_parse_json_container("{\"a\":1}").is_some());+ assert!(try_parse_json_container("[1,2,3]").is_some());+ assert!(try_parse_json_container("123").is_none()); // bare scalar+ assert!(try_parse_json_container("\"hello\"").is_none()); // bare string+ assert!(try_parse_json_container("not json").is_none());+ assert!(try_parse_json_container("{malformed").is_none());}}diff --git a/crates/headroom-core/tests/ccr_roundtrip.rs b/crates/headroom-core/tests/ccr_roundtrip.rsindex 09b140451..28d35dd8a 100644--- a/crates/headroom-core/tests/ccr_roundtrip.rs+++ b/crates/headroom-core/tests/ccr_roundtrip.rs@@ -222,3 +222,159 @@ fn full_crush_pipeline_roundtrips_through_store() {let store_len = store.len();assert!(store_len > 0, "expected at least one CCR store entry");}++// ─── PR8 additions: marker injection + walker unification ──────────++#[test]+fn lossy_crush_injects_marker_into_output_json() {+ // Cornerstone of PR8: the public `crush()` API output now carries+ // the `<<ccr:HASH ...>>` marker as a string element of the array.+ // Without this, the LLM never sees the retrieval pointer.+ let crusher = SmartCrusher::new(force_lossy_config());+ let items = lossy_friendly_items(50);+ let raw = serde_json::to_string(&Value::Array(items)).unwrap();++ let result = crusher.crush(&raw, "", 1.0);++ assert!(+ result.compressed.contains("<<ccr:"),+ "expected marker in output, got: {}",+ result.compressed+ );+ assert!(+ result.compressed.contains("rows_offloaded>>"),+ "marker should advertise dropped count: {}",+ result.compressed+ );++ // The hash in the marker is the same one in the store.+ let marker_hash =+ extract_hash_from_marker(&result.compressed).expect("marker must embed a hash");+ let store = crusher.ccr_store().unwrap();+ assert!(+ store.get(&marker_hash).is_some(),+ "marker hash {marker_hash} must resolve in the store"+ );+}++#[test]+fn nested_array_inside_object_gets_marker_injected() {+ // The public crush() recurses through wrapper objects. A nested+ // `events: [...]` array that triggers lossy must get a marker too.+ let crusher = SmartCrusher::new(force_lossy_config());+ let doc = json!({+ "user": "alice",+ "events": (0..50).map(|i| json!({"id": i, "status": "ok"}))+ .collect::<Vec<_>>(),+ });++ let result = crusher.crush(&doc.to_string(), "", 1.0);+ assert!(result.compressed.contains("<<ccr:"));++ let parsed: Value = serde_json::from_str(&result.compressed).unwrap();+ let events = parsed.get("events").expect("events field preserved");+ let arr = events.as_array().expect("events stays an array");+ let last = arr.last().expect("non-empty array");+ let marker = last+ .get("_ccr_dropped")+ .and_then(|v| v.as_str())+ .expect("sentinel object with _ccr_dropped key");+ assert!(+ marker.starts_with("<<ccr:"),+ "expected CCR marker text, got: {marker}"+ );+}++#[test]+fn opaque_string_in_object_emits_marker_and_stores_original() {+ // Walker semantics in process_value: a long base64-ish blob in+ // an object field should be replaced with a CCR marker AND the+ // original bytes stashed in the store.+ let crusher = SmartCrusher::new(SmartCrusherConfig::default());+ let store = crusher.ccr_store().unwrap().clone();+ let starting_len = store.len();++ // 64-char base64 alphabet repeated → tripping the opaque-blob+ // detector deterministically.+ let big = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=".repeat(8);+ let doc = json!({"id": 1, "blob": big.clone()});++ let result = crusher.crush(&doc.to_string(), "", 1.0);+ let parsed: Value = serde_json::from_str(&result.compressed).unwrap();+ let blob_out = parsed.get("blob").and_then(|v| v.as_str()).unwrap();++ assert!(+ blob_out.starts_with("<<ccr:") && blob_out.contains(",base64,"),+ "expected base64 CCR marker, got: {blob_out}"+ );++ // The store grew by 1 and holds the original payload.+ assert_eq!(+ store.len(),+ starting_len + 1,+ "opaque-string CCR must write to store"+ );+ let marker_hash = extract_inner_hash(blob_out).unwrap();+ let retrieved = store.get(&marker_hash).expect("hash must resolve");+ assert_eq!(retrieved, big);+}++#[test]+fn stringified_json_array_recurses_and_compacts() {+ // A field whose value is a JSON-encoded array should be parsed,+ // recursively crushed, and re-encoded — the walker behavior, now+ // available from the main pipeline.+ let crusher = SmartCrusher::new(force_lossy_config());+ let inner = (0..50)+ .map(|i| json!({"id": i, "status": "ok"}))+ .collect::<Vec<_>>();+ let inner_json = serde_json::to_string(&inner).unwrap();+ let doc = json!({"id": "outer", "payload": inner_json});++ let result = crusher.crush(&doc.to_string(), "", 1.0);+ let parsed: Value = serde_json::from_str(&result.compressed).unwrap();+ let payload = parsed.get("payload").and_then(|v| v.as_str()).unwrap();++ // The payload string went through process_value's String arm,+ // which parsed it as JSON, recursed, and re-emitted. Result:+ // either a marker-bearing array string or a direct-rendered form.+ assert!(+ payload.contains("<<ccr:") || payload.contains("rows_offloaded"),+ "expected stringified-JSON to be processed, got: {payload}"+ );+}++#[test]+fn document_walker_with_store_roundtrips_opaque_blob() {+ // Direct DocumentCompactor usage with the same store.+ use headroom_core::transforms::smart_crusher::compaction::DocumentCompactor;++ let store: Arc<dyn CcrStore> = Arc::new(InMemoryCcrStore::new());+ let dc = DocumentCompactor::new().with_ccr_store(store.clone());++ let big = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=".repeat(8);+ let out = dc.compact(json!({"id": 1, "blob": big.clone()}));+ let blob = out.pointer("/blob").and_then(|v| v.as_str()).unwrap();+ assert!(blob.starts_with("<<ccr:"));++ let h = extract_inner_hash(blob).unwrap();+ assert_eq!(store.get(&h).unwrap(), big);+}++// ─── helpers ──────────────────────────────────────────────────────++/// Pull the hash out of a `<<ccr:HASH N_rows_offloaded>>` row marker.+fn extract_hash_from_marker(s: &str) -> Option<String> {+ let i = s.find("<<ccr:")?;+ let after = &s[i + 6..];+ let end = after.find(' ')?;+ Some(after[..end].to_string())+}++/// Pull the hash out of a `<<ccr:HASH,KIND,SIZE>>` opaque marker.+fn extract_inner_hash(s: &str) -> Option<String> {+ let i = s.find("<<ccr:")?;+ let after = &s[i + 6..];+ let end = after.find(',')?;+ Some(after[..end].to_string())+}diff --git a/crates/headroom-py/src/lib.rs b/crates/headroom-py/src/lib.rsindex 70197de8e..b5b0eb36a 100644--- a/crates/headroom-py/src/lib.rs+++ b/crates/headroom-py/src/lib.rs@@ -15,6 +15,7 @@use std::collections::BTreeMap;+use headroom_core::transforms::smart_crusher::compaction::DocumentCompactor;use headroom_core::transforms::smart_crusher::{CrushResult as RustCrushResult, SmartCrusher as RustSmartCrusher,SmartCrusherConfig as RustSmartCrusherConfig,@@ -705,6 +706,30 @@ impl PySmartCrusher {)}+ /// Run the document-level walker on `doc_json` (JSON string) and+ /// return the compacted document as JSON.+ ///+ /// The walker recursively descends through objects, arrays, and+ /// strings; tabular sub-arrays become rendered CSV+schema strings,+ /// long opaque blobs become `<<ccr:HASH,KIND,SIZE>>` markers (with+ /// originals stashed in this crusher's CCR store, so `ccr_get`+ /// resolves them).+ ///+ /// Distinct from `crush_array_json`: this is the lossless walker+ /// pass without per-array lossy crushing — useful when the caller+ /// wants document-shape compaction (forms, configs, mixed records)+ /// rather than statistical row drop.+ fn compact_document_json(&self, doc_json: &str) -> String {+ let parsed: serde_json::Value =+ serde_json::from_str(doc_json).unwrap_or_else(|e| panic!("doc_json must be JSON: {e}"));+ let mut dc = DocumentCompactor::new();+ if let Some(store) = self.inner.ccr_store() {+ dc = dc.with_ccr_store(store.clone());+ }+ let out = dc.compact(parsed);+ serde_json::to_string(&out).expect("serialize compacted document")+ }+/// Look up an original payload by CCR hash.////// When the lossy path drops rows, it stashes the **full original**diff --git a/headroom/transforms/smart_crusher.py b/headroom/transforms/smart_crusher.pyindex 3dc7819c3..1d1d3abd4 100644--- a/headroom/transforms/smart_crusher.py+++ b/headroom/transforms/smart_crusher.py@@ -214,6 +214,20 @@ class SmartCrusher(Transform):result: dict[str, Any] = self._rust.crush_array_json(items_json, query, bias)return result+ def compact_document_json(self, doc_json: str) -> str:+ """Run the document walker on ``doc_json`` and return compacted JSON.++ Lossless walker pass over objects, arrays, and strings —+ tabular sub-arrays become CSV+schema strings, long opaque+ blobs become ``<<ccr:HASH,KIND,SIZE>>`` markers (originals+ stashed in this crusher's CCR store, so ``ccr_get`` resolves them).++ Use this when callers want pure document-shape compaction+ without per-array lossy crushing.+ """+ result: str = self._rust.compact_document_json(doc_json)+ return result+def ccr_get(self, hash_key: str) -> str | None:"""Look up an original payload by CCR hash from the Rust store.diff --git a/tests/parity/fixtures/smart_crusher/dict_array_100_sequential_b91d1fff5ba3.json b/tests/parity/fixtures/smart_crusher/dict_array_100_sequential_b91d1fff5ba3.jsonindex 7810da620..84899845e 100644--- a/tests/parity/fixtures/smart_crusher/dict_array_100_sequential_b91d1fff5ba3.json+++ b/tests/parity/fixtures/smart_crusher/dict_array_100_sequential_b91d1fff5ba3.json@@ -24,7 +24,7 @@"input_sha256": "b91d1fff5ba3ca82d797014b6f93c01e5e4ab7e2f3667c463a0015a2166617c3","label": "dict_array_100_sequential","output": {- "compressed": "[{\"id\":0,\"level\":\"warn\",\"message\":\"seq 0\"},{\"id\":7,\"level\":\"warn\",\"message\":\"seq 7\"},{\"id\":14,\"level\":\"warn\",\"message\":\"seq 14\"},{\"id\":21,\"level\":\"warn\",\"message\":\"seq 21\"},{\"id\":28,\"level\":\"warn\",\"message\":\"seq 28\"},{\"id\":35,\"level\":\"warn\",\"message\":\"seq 35\"},{\"id\":42,\"level\":\"warn\",\"message\":\"seq 42\"},{\"id\":49,\"level\":\"warn\",\"message\":\"seq 49\"},{\"id\":56,\"level\":\"warn\",\"message\":\"seq 56\"},{\"id\":63,\"level\":\"warn\",\"message\":\"seq 63\"},{\"id\":70,\"level\":\"warn\",\"message\":\"seq 70\"},{\"id\":77,\"level\":\"warn\",\"message\":\"seq 77\"},{\"id\":84,\"level\":\"warn\",\"message\":\"seq 84\"},{\"id\":91,\"level\":\"warn\",\"message\":\"seq 91\"},{\"id\":98,\"level\":\"warn\",\"message\":\"seq 98\"}]",+ "compressed": "[{\"id\":0,\"level\":\"warn\",\"message\":\"seq 0\"},{\"id\":7,\"level\":\"warn\",\"message\":\"seq 7\"},{\"id\":14,\"level\":\"warn\",\"message\":\"seq 14\"},{\"id\":21,\"level\":\"warn\",\"message\":\"seq 21\"},{\"id\":28,\"level\":\"warn\",\"message\":\"seq 28\"},{\"id\":35,\"level\":\"warn\",\"message\":\"seq 35\"},{\"id\":42,\"level\":\"warn\",\"message\":\"seq 42\"},{\"id\":49,\"level\":\"warn\",\"message\":\"seq 49\"},{\"id\":56,\"level\":\"warn\",\"message\":\"seq 56\"},{\"id\":63,\"level\":\"warn\",\"message\":\"seq 63\"},{\"id\":70,\"level\":\"warn\",\"message\":\"seq 70\"},{\"id\":77,\"level\":\"warn\",\"message\":\"seq 77\"},{\"id\":84,\"level\":\"warn\",\"message\":\"seq 84\"},{\"id\":91,\"level\":\"warn\",\"message\":\"seq 91\"},{\"id\":98,\"level\":\"warn\",\"message\":\"seq 98\"},{\"_ccr_dropped\":\"<<ccr:1d0dd94cf2cd 85_rows_offloaded>>\"}]","original": "[{\"id\": 0, \"level\": \"warn\", \"message\": \"seq 0\"}, {\"id\": 1, \"level\": \"info\", \"message\": \"seq 1\"}, {\"id\": 2, \"level\": \"info\", \"message\": \"seq 2\"}, {\"id\": 3, \"level\": \"info\", \"message\": \"seq 3\"}, {\"id\": 4, \"level\": \"info\", \"message\": \"seq 4\"}, {\"id\": 5, \"level\": \"info\", \"message\": \"seq 5\"}, {\"id\": 6, \"level\": \"info\", \"message\": \"seq 6\"}, {\"id\": 7, \"level\": \"warn\", \"message\": \"seq 7\"}, {\"id\": 8, \"level\": \"info\", \"message\": \"seq 8\"}, {\"id\": 9, \"level\": \"info\", \"message\": \"seq 9\"}, {\"id\": 10, \"level\": \"info\", \"message\": \"seq 10\"}, {\"id\": 11, \"level\": \"info\", \"message\": \"seq 11\"}, {\"id\": 12, \"level\": \"info\", \"message\": \"seq 12\"}, {\"id\": 13, \"level\": \"info\", \"message\": \"seq 13\"}, {\"id\": 14, \"level\": \"warn\", \"message\": \"seq 14\"}, {\"id\": 15, \"level\": \"info\", \"message\": \"seq 15\"}, {\"id\": 16, \"level\": \"info\", \"message\": \"seq 16\"}, {\"id\": 17, \"level\": \"info\", \"message\": \"seq 17\"}, {\"id\": 18, \"level\": \"info\", \"message\": \"seq 18\"}, {\"id\": 19, \"level\": \"info\", \"message\": \"seq 19\"}, {\"id\": 20, \"level\": \"info\", \"message\": \"seq 20\"}, {\"id\": 21, \"level\": \"warn\", \"message\": \"seq 21\"}, {\"id\": 22, \"level\": \"info\", \"message\": \"seq 22\"}, {\"id\": 23, \"level\": \"info\", \"message\": \"seq 23\"}, {\"id\": 24, \"level\": \"info\", \"message\": \"seq 24\"}, {\"id\": 25, \"level\": \"info\", \"message\": \"seq 25\"}, {\"id\": 26, \"level\": \"info\", \"message\": \"seq 26\"}, {\"id\": 27, \"level\": \"info\", \"message\": \"seq 27\"}, {\"id\": 28, \"level\": \"warn\", \"message\": \"seq 28\"}, {\"id\": 29, \"level\": \"info\", \"message\": \"seq 29\"}, {\"id\": 30, \"level\": \"info\", \"message\": \"seq 30\"}, {\"id\": 31, \"level\": \"info\", \"message\": \"seq 31\"}, {\"id\": 32, \"level\": \"info\", \"message\": \"seq 32\"}, {\"id\": 33, \"level\": \"info\", \"message\": \"seq 33\"}, {\"id\": 34, \"level\": \"info\", \"message\": \"seq 34\"}, {\"id\": 35, \"level\": \"warn\", \"message\": \"seq 35\"}, {\"id\": 36, \"level\": \"info\", \"message\": \"seq 36\"}, {\"id\": 37, \"level\": \"info\", \"message\": \"seq 37\"}, {\"id\": 38, \"level\": \"info\", \"message\": \"seq 38\"}, {\"id\": 39, \"level\": \"info\", \"message\": \"seq 39\"}, {\"id\": 40, \"level\": \"info\", \"message\": \"seq 40\"}, {\"id\": 41, \"level\": \"info\", \"message\": \"seq 41\"}, {\"id\": 42, \"level\": \"warn\", \"message\": \"seq 42\"}, {\"id\": 43, \"level\": \"info\", \"message\": \"seq 43\"}, {\"id\": 44, \"level\": \"info\", \"message\": \"seq 44\"}, {\"id\": 45, \"level\": \"info\", \"message\": \"seq 45\"}, {\"id\": 46, \"level\": \"info\", \"message\": \"seq 46\"}, {\"id\": 47, \"level\": \"info\", \"message\": \"seq 47\"}, {\"id\": 48, \"level\": \"info\", \"message\": \"seq 48\"}, {\"id\": 49, \"level\": \"warn\", \"message\": \"seq 49\"}, {\"id\": 50, \"level\": \"info\", \"message\": \"seq 50\"}, {\"id\": 51, \"level\": \"info\", \"message\": \"seq 51\"}, {\"id\": 52, \"level\": \"info\", \"message\": \"seq 52\"}, {\"id\": 53, \"level\": \"info\", \"message\": \"seq 53\"}, {\"id\": 54, \"level\": \"info\", \"message\": \"seq 54\"}, {\"id\": 55, \"level\": \"info\", \"message\": \"seq 55\"}, {\"id\": 56, \"level\": \"warn\", \"message\": \"seq 56\"}, {\"id\": 57, \"level\": \"info\", \"message\": \"seq 57\"}, {\"id\": 58, \"level\": \"info\", \"message\": \"seq 58\"}, {\"id\": 59, \"level\": \"info\", \"message\": \"seq 59\"}, {\"id\": 60, \"level\": \"info\", \"message\": \"seq 60\"}, {\"id\": 61, \"level\": \"info\", \"message\": \"seq 61\"}, {\"id\": 62, \"level\": \"info\", \"message\": \"seq 62\"}, {\"id\": 63, \"level\": \"warn\", \"message\": \"seq 63\"}, {\"id\": 64, \"level\": \"info\", \"message\": \"seq 64\"}, {\"id\": 65, \"level\": \"info\", \"message\": \"seq 65\"}, {\"id\": 66, \"level\": \"info\", \"message\": \"seq 66\"}, {\"id\": 67, \"level\": \"info\", \"message\": \"seq 67\"}, {\"id\": 68, \"level\": \"info\", \"message\": \"seq 68\"}, {\"id\": 69, \"level\": \"info\", \"message\": \"seq 69\"}, {\"id\": 70, \"level\": \"warn\", \"message\": \"seq 70\"}, {\"id\": 71, \"level\": \"info\", \"message\": \"seq 71\"}, {\"id\": 72, \"level\": \"info\", \"message\": \"seq 72\"}, {\"id\": 73, \"level\": \"info\", \"message\": \"seq 73\"}, {\"id\": 74, \"level\": \"info\", \"message\": \"seq 74\"}, {\"id\": 75, \"level\": \"info\", \"message\": \"seq 75\"}, {\"id\": 76, \"level\": \"info\", \"message\": \"seq 76\"}, {\"id\": 77, \"level\": \"warn\", \"message\": \"seq 77\"}, {\"id\": 78, \"level\": \"info\", \"message\": \"seq 78\"}, {\"id\": 79, \"level\": \"info\", \"message\": \"seq 79\"}, {\"id\": 80, \"level\": \"info\", \"message\": \"seq 80\"}, {\"id\": 81, \"level\": \"info\", \"message\": \"seq 81\"}, {\"id\": 82, \"level\": \"info\", \"message\": \"seq 82\"}, {\"id\": 83, \"level\": \"info\", \"message\": \"seq 83\"}, {\"id\": 84, \"level\": \"warn\", \"message\": \"seq 84\"}, {\"id\": 85, \"level\": \"info\", \"message\": \"seq 85\"}, {\"id\": 86, \"level\": \"info\", \"message\": \"seq 86\"}, {\"id\": 87, \"level\": \"info\", \"message\": \"seq 87\"}, {\"id\": 88, \"level\": \"info\", \"message\": \"seq 88\"}, {\"id\": 89, \"level\": \"info\", \"message\": \"seq 89\"}, {\"id\": 90, \"level\": \"info\", \"message\": \"seq 90\"}, {\"id\": 91, \"level\": \"warn\", \"message\": \"seq 91\"}, {\"id\": 92, \"level\": \"info\", \"message\": \"seq 92\"}, {\"id\": 93, \"level\": \"info\", \"message\": \"seq 93\"}, {\"id\": 94, \"level\": \"info\", \"message\": \"seq 94\"}, {\"id\": 95, \"level\": \"info\", \"message\": \"seq 95\"}, {\"id\": 96, \"level\": \"info\", \"message\": \"seq 96\"}, {\"id\": 97, \"level\": \"info\", \"message\": \"seq 97\"}, {\"id\": 98, \"level\": \"warn\", \"message\": \"seq 98\"}, {\"id\": 99, \"level\": \"info\", \"message\": \"seq 99\"}]","strategy": "smart_sample(100->15)","was_modified": truediff --git a/tests/parity/fixtures/smart_crusher/dict_array_30_b0ba1d26fd03.json b/tests/parity/fixtures/smart_crusher/dict_array_30_b0ba1d26fd03.jsonindex 8da36040d..77291ad48 100644--- a/tests/parity/fixtures/smart_crusher/dict_array_30_b0ba1d26fd03.json+++ b/tests/parity/fixtures/smart_crusher/dict_array_30_b0ba1d26fd03.json@@ -24,7 +24,7 @@"input_sha256": "b0ba1d26fd03d9b5010ffd8d4c6185e0c5baf03ab783316f3ad15830d1bba779","label": "dict_array_30","output": {- "compressed": "[{\"id\":0,\"status\":\"error\",\"msg\":\"line 0\"},{\"id\":1,\"status\":\"ok\",\"msg\":\"line 1\"},{\"id\":2,\"status\":\"ok\",\"msg\":\"line 2\"},{\"id\":4,\"status\":\"ok\",\"msg\":\"line 4\"},{\"id\":5,\"status\":\"error\",\"msg\":\"line 5\"},{\"id\":7,\"status\":\"ok\",\"msg\":\"line 7\"},{\"id\":9,\"status\":\"ok\",\"msg\":\"line 9\"},{\"id\":10,\"status\":\"error\",\"msg\":\"line 10\"},{\"id\":12,\"status\":\"ok\",\"msg\":\"line 12\"},{\"id\":14,\"status\":\"ok\",\"msg\":\"line 14\"},{\"id\":15,\"status\":\"error\",\"msg\":\"line 15\"},{\"id\":17,\"status\":\"ok\",\"msg\":\"line 17\"},{\"id\":20,\"status\":\"error\",\"msg\":\"line 20\"},{\"id\":25,\"status\":\"error\",\"msg\":\"line 25\"},{\"id\":29,\"status\":\"ok\",\"msg\":\"line 29\"}]",+ "compressed": "[{\"id\":0,\"status\":\"error\",\"msg\":\"line 0\"},{\"id\":1,\"status\":\"ok\",\"msg\":\"line 1\"},{\"id\":2,\"status\":\"ok\",\"msg\":\"line 2\"},{\"id\":4,\"status\":\"ok\",\"msg\":\"line 4\"},{\"id\":5,\"status\":\"error\",\"msg\":\"line 5\"},{\"id\":7,\"status\":\"ok\",\"msg\":\"line 7\"},{\"id\":9,\"status\":\"ok\",\"msg\":\"line 9\"},{\"id\":10,\"status\":\"error\",\"msg\":\"line 10\"},{\"id\":12,\"status\":\"ok\",\"msg\":\"line 12\"},{\"id\":14,\"status\":\"ok\",\"msg\":\"line 14\"},{\"id\":15,\"status\":\"error\",\"msg\":\"line 15\"},{\"id\":17,\"status\":\"ok\",\"msg\":\"line 17\"},{\"id\":20,\"status\":\"error\",\"msg\":\"line 20\"},{\"id\":25,\"status\":\"error\",\"msg\":\"line 25\"},{\"id\":29,\"status\":\"ok\",\"msg\":\"line 29\"},{\"_ccr_dropped\":\"<<ccr:89f81e97033e 15_rows_offloaded>>\"}]","original": "[{\"id\": 0, \"status\": \"error\", \"msg\": \"line 0\"}, {\"id\": 1, \"status\": \"ok\", \"msg\": \"line 1\"}, {\"id\": 2, \"status\": \"ok\", \"msg\": \"line 2\"}, {\"id\": 3, \"status\": \"ok\", \"msg\": \"line 3\"}, {\"id\": 4, \"status\": \"ok\", \"msg\": \"line 4\"}, {\"id\": 5, \"status\": \"error\", \"msg\": \"line 5\"}, {\"id\": 6, \"status\": \"ok\", \"msg\": \"line 6\"}, {\"id\": 7, \"status\": \"ok\", \"msg\": \"line 7\"}, {\"id\": 8, \"status\": \"ok\", \"msg\": \"line 8\"}, {\"id\": 9, \"status\": \"ok\", \"msg\": \"line 9\"}, {\"id\": 10, \"status\": \"error\", \"msg\": \"line 10\"}, {\"id\": 11, \"status\": \"ok\", \"msg\": \"line 11\"}, {\"id\": 12, \"status\": \"ok\", \"msg\": \"line 12\"}, {\"id\": 13, \"status\": \"ok\", \"msg\": \"line 13\"}, {\"id\": 14, \"status\": \"ok\", \"msg\": \"line 14\"}, {\"id\": 15, \"status\": \"error\", \"msg\": \"line 15\"}, {\"id\": 16, \"status\": \"ok\", \"msg\": \"line 16\"}, {\"id\": 17, \"status\": \"ok\", \"msg\": \"line 17\"}, {\"id\": 18, \"status\": \"ok\", \"msg\": \"line 18\"}, {\"id\": 19, \"status\": \"ok\", \"msg\": \"line 19\"}, {\"id\": 20, \"status\": \"error\", \"msg\": \"line 20\"}, {\"id\": 21, \"status\": \"ok\", \"msg\": \"line 21\"}, {\"id\": 22, \"status\": \"ok\", \"msg\": \"line 22\"}, {\"id\": 23, \"status\": \"ok\", \"msg\": \"line 23\"}, {\"id\": 24, \"status\": \"ok\", \"msg\": \"line 24\"}, {\"id\": 25, \"status\": \"error\", \"msg\": \"line 25\"}, {\"id\": 26, \"status\": \"ok\", \"msg\": \"line 26\"}, {\"id\": 27, \"status\": \"ok\", \"msg\": \"line 27\"}, {\"id\": 28, \"status\": \"ok\", \"msg\": \"line 28\"}, {\"id\": 29, \"status\": \"ok\", \"msg\": \"line 29\"}]","strategy": "smart_sample(30->15)","was_modified": truediff --git a/tests/parity/fixtures/smart_crusher/dict_array_30_bias_high_dc3b36e60560.json b/tests/parity/fixtures/smart_crusher/dict_array_30_bias_high_dc3b36e60560.jsonindex 9dbeb71d7..50241a4e3 100644--- a/tests/parity/fixtures/smart_crusher/dict_array_30_bias_high_dc3b36e60560.json+++ b/tests/parity/fixtures/smart_crusher/dict_array_30_bias_high_dc3b36e60560.json@@ -24,7 +24,7 @@"input_sha256": "dc3b36e605600ba3d7b91fae6a05d2fb3c849de875c455530d61ed40bbb2c956","label": "dict_array_30_bias_high","output": {- "compressed": "[{\"id\":0,\"status\":\"error\",\"msg\":\"line 0\"},{\"id\":1,\"status\":\"ok\",\"msg\":\"line 1\"},{\"id\":2,\"status\":\"ok\",\"msg\":\"line 2\"},{\"id\":4,\"status\":\"ok\",\"msg\":\"line 4\"},{\"id\":5,\"status\":\"error\",\"msg\":\"line 5\"},{\"id\":7,\"status\":\"ok\",\"msg\":\"line 7\"},{\"id\":9,\"status\":\"ok\",\"msg\":\"line 9\"},{\"id\":10,\"status\":\"error\",\"msg\":\"line 10\"},{\"id\":12,\"status\":\"ok\",\"msg\":\"line 12\"},{\"id\":14,\"status\":\"ok\",\"msg\":\"line 14\"},{\"id\":15,\"status\":\"error\",\"msg\":\"line 15\"},{\"id\":17,\"status\":\"ok\",\"msg\":\"line 17\"},{\"id\":20,\"status\":\"error\",\"msg\":\"line 20\"},{\"id\":25,\"status\":\"error\",\"msg\":\"line 25\"},{\"id\":29,\"status\":\"ok\",\"msg\":\"line 29\"}]",+ "compressed": "[{\"id\":0,\"status\":\"error\",\"msg\":\"line 0\"},{\"id\":1,\"status\":\"ok\",\"msg\":\"line 1\"},{\"id\":2,\"status\":\"ok\",\"msg\":\"line 2\"},{\"id\":4,\"status\":\"ok\",\"msg\":\"line 4\"},{\"id\":5,\"status\":\"error\",\"msg\":\"line 5\"},{\"id\":7,\"status\":\"ok\",\"msg\":\"line 7\"},{\"id\":9,\"status\":\"ok\",\"msg\":\"line 9\"},{\"id\":10,\"status\":\"error\",\"msg\":\"line 10\"},{\"id\":12,\"status\":\"ok\",\"msg\":\"line 12\"},{\"id\":14,\"status\":\"ok\",\"msg\":\"line 14\"},{\"id\":15,\"status\":\"error\",\"msg\":\"line 15\"},{\"id\":17,\"status\":\"ok\",\"msg\":\"line 17\"},{\"id\":20,\"status\":\"error\",\"msg\":\"line 20\"},{\"id\":25,\"status\":\"error\",\"msg\":\"line 25\"},{\"id\":29,\"status\":\"ok\",\"msg\":\"line 29\"},{\"_ccr_dropped\":\"<<ccr:89f81e97033e 15_rows_offloaded>>\"}]","original": "[{\"id\": 0, \"status\": \"error\", \"msg\": \"line 0\"}, {\"id\": 1, \"status\": \"ok\", \"msg\": \"line 1\"}, {\"id\": 2, \"status\": \"ok\", \"msg\": \"line 2\"}, {\"id\": 3, \"status\": \"ok\", \"msg\": \"line 3\"}, {\"id\": 4, \"status\": \"ok\", \"msg\": \"line 4\"}, {\"id\": 5, \"status\": \"error\", \"msg\": \"line 5\"}, {\"id\": 6, \"status\": \"ok\", \"msg\": \"line 6\"}, {\"id\": 7, \"status\": \"ok\", \"msg\": \"line 7\"}, {\"id\": 8, \"status\": \"ok\", \"msg\": \"line 8\"}, {\"id\": 9, \"status\": \"ok\", \"msg\": \"line 9\"}, {\"id\": 10, \"status\": \"error\", \"msg\": \"line 10\"}, {\"id\": 11, \"status\": \"ok\", \"msg\": \"line 11\"}, {\"id\": 12, \"status\": \"ok\", \"msg\": \"line 12\"}, {\"id\": 13, \"status\": \"ok\", \"msg\": \"line 13\"}, {\"id\": 14, \"status\": \"ok\", \"msg\": \"line 14\"}, {\"id\": 15, \"status\": \"error\", \"msg\": \"line 15\"}, {\"id\": 16, \"status\": \"ok\", \"msg\": \"line 16\"}, {\"id\": 17, \"status\": \"ok\", \"msg\": \"line 17\"}, {\"id\": 18, \"status\": \"ok\", \"msg\": \"line 18\"}, {\"id\": 19, \"status\": \"ok\", \"msg\": \"line 19\"}, {\"id\": 20, \"status\": \"error\", \"msg\": \"line 20\"}, {\"id\": 21, \"status\": \"ok\", \"msg\": \"line 21\"}, {\"id\": 22, \"status\": \"ok\", \"msg\": \"line 22\"}, {\"id\": 23, \"status\": \"ok\", \"msg\": \"line 23\"}, {\"id\": 24, \"status\": \"ok\", \"msg\": \"line 24\"}, {\"id\": 25, \"status\": \"error\", \"msg\": \"line 25\"}, {\"id\": 26, \"status\": \"ok\", \"msg\": \"line 26\"}, {\"id\": 27, \"status\": \"ok\", \"msg\": \"line 27\"}, {\"id\": 28, \"status\": \"ok\", \"msg\": \"line 28\"}, {\"id\": 29, \"status\": \"ok\", \"msg\": \"line 29\"}]","strategy": "smart_sample(30->15)","was_modified": truediff --git a/tests/parity/fixtures/smart_crusher/dict_array_30_bias_low_a850512901bd.json b/tests/parity/fixtures/smart_crusher/dict_array_30_bias_low_a850512901bd.jsonindex 091b94ec6..0e5022bfa 100644--- a/tests/parity/fixtures/smart_crusher/dict_array_30_bias_low_a850512901bd.json+++ b/tests/parity/fixtures/smart_crusher/dict_array_30_bias_low_a850512901bd.json@@ -24,7 +24,7 @@"input_sha256": "a850512901bdb42373b58e753e7a9ed2ae20fab671f686f2fd7a1c670889eb93","label": "dict_array_30_bias_low","output": {- "compressed": "[{\"id\":0,\"status\":\"error\",\"msg\":\"line 0\"},{\"id\":1,\"status\":\"ok\",\"msg\":\"line 1\"},{\"id\":2,\"status\":\"ok\",\"msg\":\"line 2\"},{\"id\":4,\"status\":\"ok\",\"msg\":\"line 4\"},{\"id\":5,\"status\":\"error\",\"msg\":\"line 5\"},{\"id\":7,\"status\":\"ok\",\"msg\":\"line 7\"},{\"id\":9,\"status\":\"ok\",\"msg\":\"line 9\"},{\"id\":10,\"status\":\"error\",\"msg\":\"line 10\"},{\"id\":12,\"status\":\"ok\",\"msg\":\"line 12\"},{\"id\":14,\"status\":\"ok\",\"msg\":\"line 14\"},{\"id\":15,\"status\":\"error\",\"msg\":\"line 15\"},{\"id\":17,\"status\":\"ok\",\"msg\":\"line 17\"},{\"id\":20,\"status\":\"error\",\"msg\":\"line 20\"},{\"id\":25,\"status\":\"error\",\"msg\":\"line 25\"},{\"id\":29,\"status\":\"ok\",\"msg\":\"line 29\"}]",+ "compressed": "[{\"id\":0,\"status\":\"error\",\"msg\":\"line 0\"},{\"id\":1,\"status\":\"ok\",\"msg\":\"line 1\"},{\"id\":2,\"status\":\"ok\",\"msg\":\"line 2\"},{\"id\":4,\"status\":\"ok\",\"msg\":\"line 4\"},{\"id\":5,\"status\":\"error\",\"msg\":\"line 5\"},{\"id\":7,\"status\":\"ok\",\"msg\":\"line 7\"},{\"id\":9,\"status\":\"ok\",\"msg\":\"line 9\"},{\"id\":10,\"status\":\"error\",\"msg\":\"line 10\"},{\"id\":12,\"status\":\"ok\",\"msg\":\"line 12\"},{\"id\":14,\"status\":\"ok\",\"msg\":\"line 14\"},{\"id\":15,\"status\":\"error\",\"msg\":\"line 15\"},{\"id\":17,\"status\":\"ok\",\"msg\":\"line 17\"},{\"id\":20,\"status\":\"error\",\"msg\":\"line 20\"},{\"id\":25,\"status\":\"error\",\"msg\":\"line 25\"},{\"id\":29,\"status\":\"ok\",\"msg\":\"line 29\"},{\"_ccr_dropped\":\"<<ccr:89f81e97033e 15_rows_offloaded>>\"}]","original": "[{\"id\": 0, \"status\": \"error\", \"msg\": \"line 0\"}, {\"id\": 1, \"status\": \"ok\", \"msg\": \"line 1\"}, {\"id\": 2, \"status\": \"ok\", \"msg\": \"line 2\"}, {\"id\": 3, \"status\": \"ok\", \"msg\": \"line 3\"}, {\"id\": 4, \"status\": \"ok\", \"msg\": \"line 4\"}, {\"id\": 5, \"status\": \"error\", \"msg\": \"line 5\"}, {\"id\": 6, \"status\": \"ok\", \"msg\": \"line 6\"}, {\"id\": 7, \"status\": \"ok\", \"msg\": \"line 7\"}, {\"id\": 8, \"status\": \"ok\", \"msg\": \"line 8\"}, {\"id\": 9, \"status\": \"ok\", \"msg\": \"line 9\"}, {\"id\": 10, \"status\": \"error\", \"msg\": \"line 10\"}, {\"id\": 11, \"status\": \"ok\", \"msg\": \"line 11\"}, {\"id\": 12, \"status\": \"ok\", \"msg\": \"line 12\"}, {\"id\": 13, \"status\": \"ok\", \"msg\": \"line 13\"}, {\"id\": 14, \"status\": \"ok\", \"msg\": \"line 14\"}, {\"id\": 15, \"status\": \"error\", \"msg\": \"line 15\"}, {\"id\": 16, \"status\": \"ok\", \"msg\": \"line 16\"}, {\"id\": 17, \"status\": \"ok\", \"msg\": \"line 17\"}, {\"id\": 18, \"status\": \"ok\", \"msg\": \"line 18\"}, {\"id\": 19, \"status\": \"ok\", \"msg\": \"line 19\"}, {\"id\": 20, \"status\": \"error\", \"msg\": \"line 20\"}, {\"id\": 21, \"status\": \"ok\", \"msg\": \"line 21\"}, {\"id\": 22, \"status\": \"ok\", \"msg\": \"line 22\"}, {\"id\": 23, \"status\": \"ok\", \"msg\": \"line 23\"}, {\"id\": 24, \"status\": \"ok\", \"msg\": \"line 24\"}, {\"id\": 25, \"status\": \"error\", \"msg\": \"line 25\"}, {\"id\": 26, \"status\": \"ok\", \"msg\": \"line 26\"}, {\"id\": 27, \"status\": \"ok\", \"msg\": \"line 27\"}, {\"id\": 28, \"status\": \"ok\", \"msg\": \"line 28\"}, {\"id\": 29, \"status\": \"ok\", \"msg\": \"line 29\"}]","strategy": "smart_sample(30->15)","was_modified": truediff --git a/tests/parity/fixtures/smart_crusher/duplicate_dicts_40_40a0670dec17.json b/tests/parity/fixtures/smart_crusher/duplicate_dicts_40_40a0670dec17.jsonindex e1d0e4c91..e8a2a55a5 100644--- a/tests/parity/fixtures/smart_crusher/duplicate_dicts_40_40a0670dec17.json+++ b/tests/parity/fixtures/smart_crusher/duplicate_dicts_40_40a0670dec17.json@@ -24,7 +24,7 @@"input_sha256": "40a0670dec17ae0f765b640ffea2a78c2f2526c03877dcabf0b6108b608ada8a","label": "duplicate_dicts_40","output": {- "compressed": "[{\"event\":\"heartbeat\",\"ok\":true}]",+ "compressed": "[{\"event\":\"heartbeat\",\"ok\":true},{\"_ccr_dropped\":\"<<ccr:6155f8b8951e 39_rows_offloaded>>\"}]","original": "[{\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}, {\"event\": \"heartbeat\", \"ok\": true}]","strategy": "smart_sample(40->1)","was_modified": truediff --git a/tests/test_transforms/test_smart_crusher_ccr_roundtrip.py b/tests/test_transforms/test_smart_crusher_ccr_roundtrip.pyindex 69018fe04..6cebf15e9 100644--- a/tests/test_transforms/test_smart_crusher_ccr_roundtrip.py+++ b/tests/test_transforms/test_smart_crusher_ccr_roundtrip.py@@ -239,6 +239,87 @@ def test_kept_subset_is_subset_of_original() -> None:assert retrieved == original+def test_marker_visible_in_crush_output_native() -> None:+ """PR8 cornerstone: the public crush() output now carries the+ `<<ccr:HASH ...>>` marker so the LLM sees the retrieval pointer."""+ from headroom._core import SmartCrusher, SmartCrusherConfig++ crusher = SmartCrusher(SmartCrusherConfig(lossless_min_savings_ratio=0.99))+ items = [{"id": i, "status": "ok"} for i in range(50)]+ raw = json.dumps(items)++ result = crusher.crush(raw, "", 1.0)+ assert "<<ccr:" in result.compressed, f"expected marker in output: {result.compressed[:200]!r}"+ assert "rows_offloaded" in result.compressed++ # The marker hash resolves in the store.+ import re++ m = re.search(r"<<ccr:([a-f0-9]+) ", result.compressed)+ assert m is not None+ hash_key = m.group(1)+ assert crusher.ccr_get(hash_key) is not None+++def test_opaque_blob_in_object_emits_marker_and_stores_native() -> None:+ """A long base64-ish blob in a field becomes a CCR marker AND the+ original gets stashed."""+ from headroom._core import SmartCrusher++ crusher = SmartCrusher()+ big = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=" * 8+ raw = json.dumps({"id": 1, "blob": big})++ result = crusher.crush(raw, "", 1.0)+ parsed = json.loads(result.compressed)+ blob_out = parsed["blob"]+ assert blob_out.startswith("<<ccr:") and ",base64," in blob_out++ # The store grew, hash resolves, original byte-equal.+ import re++ m = re.search(r"<<ccr:([a-f0-9]+),", blob_out)+ assert m is not None+ retrieved = crusher.ccr_get(m.group(1))+ assert retrieved == big+++def test_compact_document_json_via_pyo3() -> None:+ """The walker is reachable from Python and writes to the same store."""+ from headroom._core import SmartCrusher++ crusher = SmartCrusher()+ starting = crusher.ccr_len()+ big = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=" * 8+ doc = {"id": 1, "blob": big}++ out = crusher.compact_document_json(json.dumps(doc))+ parsed = json.loads(out)+ assert parsed["id"] == 1+ assert parsed["blob"].startswith("<<ccr:")+ assert crusher.ccr_len() == starting + 1++ import re++ m = re.search(r"<<ccr:([a-f0-9]+),", parsed["blob"])+ assert crusher.ccr_get(m.group(1)) == big+++def test_compact_document_via_shim() -> None:+ """Same path via the Python shim."""+ from headroom.config import SmartCrusherConfig as PyConfig+ from headroom.transforms.smart_crusher import SmartCrusher++ crusher = SmartCrusher(PyConfig())+ items = [{"id": i, "status": "ok", "tag": "alpha"} for i in range(30)]+ out = crusher.compact_document_json(json.dumps({"events": items}))+ parsed = json.loads(out)+ # Tabular sub-array compacted to a string.+ assert isinstance(parsed["events"], str), (+ f"expected string, got {type(parsed['events']).__name__}"+ )++def test_distinct_payloads_have_distinct_hashes_and_separate_storage() -> None:"""Two different payloads → two different hashes → bothindependently retrievable. Pins the per-payload isolation."""