Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(smart-crusher): disclose dropped scalar-array items with a retrievable ccr sentinel (#3692)

Raphael Malikianc1fc84ade6c8
diff --git a/crates/headroom-core/src/transforms/smart_crusher/crusher.rs b/crates/headroom-core/src/transforms/smart_crusher/crusher.rs
index 0dc34df77..9fb40c9e7 100644
--- a/crates/headroom-core/src/transforms/smart_crusher/crusher.rs
+++ b/crates/headroom-core/src/transforms/smart_crusher/crusher.rs
@@ -595,20 +595,31 @@ impl SmartCrusher {
let strs: Vec<&str> = arr.iter().filter_map(|v| v.as_str()).collect();
let (crushed, strategy) = crush_string_array(&strs, &self.config, bias);
info_parts.push(format!("{}({}->{})", strategy, n, crushed.len()));
- let crushed_values: Vec<Value> =
+ let mut crushed_values: Vec<Value> =
crushed.into_iter().map(Value::String).collect();
+ // #3650: dropped strings get a retrievable
+ // sentinel — silent truncation is a data-loss
+ // bug even when the marker gate is on.
+ self.append_scalar_drop_sentinel(&mut crushed_values, arr);
return (Value::Array(crushed_values), info_parts.join(","));
}
ArrayType::NumberArray => {
let (crushed, strategy) = crush_number_array(arr, &self.config, bias);
info_parts.push(format!("{}({}->{})", strategy, n, crushed.len()));
- return (Value::Array(crushed), info_parts.join(","));
+ let mut crushed_values = crushed;
+ // #3650: same sentinel contract as StringArray.
+ self.append_scalar_drop_sentinel(&mut crushed_values, arr);
+ return (Value::Array(crushed_values), info_parts.join(","));
}
ArrayType::MixedArray => {
let (crushed, strategy) =
self.crush_mixed_array(arr, query_context, bias);
info_parts.push(format!("{}({}->{})", strategy, n, crushed.len()));
- return (Value::Array(crushed), info_parts.join(","));
+ let mut crushed_values = crushed;
+ // #3650: same sentinel contract as the
+ // StringArray/NumberArray arms.
+ self.append_scalar_drop_sentinel(&mut crushed_values, arr);
+ return (Value::Array(crushed_values), info_parts.join(","));
}
// NestedArray, BoolArray, Empty → fall through
// to recursive descent.
@@ -1107,6 +1118,30 @@ impl SmartCrusher {
);
(result, strategy)
}
+
+ /// Appends a visible, retrievable CCR sentinel to `kept` when items were
+ /// dropped from `original` and the marker gate is on.
+ ///
+ /// Mirrors the dict-array path: the full original array is serialized
+ /// once (`canonical_array_json`), hashed with the shared
+ /// `hash_canonical` (SHA256, first 12 hex chars — the same scheme the
+ /// Python bridge and retrieval use), and stored in the CCR store so
+ /// the marker round-trips through retrieval. No-op when nothing was
+ /// dropped or `enable_ccr_marker` is false.
+ fn append_scalar_drop_sentinel(&self, kept: &mut Vec<Value>, original: &[Value]) {
+ let dropped = original.len().saturating_sub(kept.len());
+ if dropped == 0 || !self.config.enable_ccr_marker {
+ return;
+ }
+ let canonical = canonical_array_json(original);
+ let hash = hash_canonical(&canonical);
+ if let Some(store) = &self.ccr_store {
+ store.put(&hash, &canonical);
+ }
+ kept.push(Value::String(format!(
+ "… {dropped} more items <<ccr:{hash} {dropped}_items_offloaded>>"
+ )));
+ }
}
// ---------- helpers ----------
@@ -1275,6 +1310,166 @@ mod tests {
use super::*;
use serde_json::json;
+ /// Extracts the CCR hash from a scalar drop sentinel produced by
+ /// `append_scalar_drop_sentinel`.
+ fn scalar_sentinel_hash(s: &str) -> &str {
+ let start = s.find("<<ccr:").expect("sentinel must carry a CCR marker") + 6;
+ let end = s[start..].find(' ').expect("marker must have a tag") + start;
+ &s[start..end]
+ }
+
+ /// Crushes a JSON array string and returns the parsed compressed array.
+ fn crush_array_values(crusher: &SmartCrusher, items: &[Value]) -> Vec<Value> {
+ let out = crusher.crush(&serde_json::to_string(items).unwrap(), "", 1.0);
+ serde_json::from_str::<Value>(&out.compressed)
+ .expect("compressed output is JSON")
+ .as_array()
+ .expect("output stays an array")
+ .clone()
+ }
+
+ /// Looks up the stored original array for a sentinel's CCR hash.
+ fn stored_original(crusher: &SmartCrusher, sentinel: &str) -> Vec<Value> {
+ let hash = scalar_sentinel_hash(sentinel);
+ let raw = crusher
+ .ccr_store()
+ .and_then(|store| store.get(hash))
+ .expect("sentinel hash must be in the CCR store");
+ serde_json::from_str(&raw).expect("stored payload is a JSON array")
+ }
+
+ // Regression tests for headroomlabs-ai/headroom#3650: scalar
+ // (string/number/mixed) array crushers dropped items silently. They
+ // must now emit a visible, retrievable CCR sentinel.
+
+ #[test]
+ fn string_array_drops_emit_retrievable_sentinel() {
+ // The issue's repro shape: 120 short slugs.
+ let items: Vec<Value> = (0..120).map(|i| json!(format!("r{i}"))).collect();
+ let config = SmartCrusherConfig {
+ enable_ccr_marker: true,
+ ..Default::default()
+ };
+ let crusher = SmartCrusher::new(config);
+
+ let kept = crush_array_values(&crusher, &items);
+ // `kept` includes the trailing sentinel, so the crusher's drop
+ // count is original minus the data items (all but the last).
+ let dropped = items.len() - (kept.len() - 1);
+ assert!(
+ kept.len() < items.len(),
+ "expected drops for a 120-item string array"
+ );
+ let sentinel = kept
+ .last()
+ .and_then(|v| v.as_str())
+ .expect("last element is the string sentinel");
+ assert!(
+ sentinel.starts_with(&format!("… {dropped} more items ")),
+ "sentinel shows the visible drop count, got: {sentinel}"
+ );
+ assert!(
+ sentinel.contains("<<ccr:")
+ && sentinel.ends_with(&format!("{dropped}_items_offloaded>>")),
+ "sentinel carries a CCR marker, got: {sentinel}"
+ );
+ // The marker hash must retrieve the FULL original array.
+ let stored = stored_original(&crusher, sentinel);
+ assert_eq!(stored.len(), 120);
+ assert_eq!(stored[119], json!("r119"));
+ }
+
+ #[test]
+ fn string_array_sentinel_suppressed_when_marker_gate_off() {
+ let items: Vec<Value> = (0..120).map(|i| json!(format!("r{i}"))).collect();
+ let config = SmartCrusherConfig {
+ enable_ccr_marker: false,
+ ..Default::default()
+ };
+ let crusher = SmartCrusher::new(config);
+
+ let kept = crush_array_values(&crusher, &items);
+ assert!(
+ !kept
+ .iter()
+ .any(|v| v.as_str().is_some_and(|s| s.contains("<<ccr:"))),
+ "no sentinel when the marker gate is off"
+ );
+ }
+
+ #[test]
+ fn string_array_no_sentinel_when_nothing_dropped() {
+ let items: Vec<Value> = (0..5).map(|i| json!(format!("r{i}"))).collect();
+ let crusher = SmartCrusher::new(SmartCrusherConfig::default());
+
+ let kept = crush_array_values(&crusher, &items);
+ assert_eq!(kept.len(), items.len());
+ assert!(
+ !kept
+ .iter()
+ .any(|v| v.as_str().is_some_and(|s| s.contains("<<ccr:"))),
+ "no sentinel when nothing was dropped"
+ );
+ }
+
+ #[test]
+ fn number_array_drops_emit_retrievable_sentinel() {
+ let items: Vec<Value> = (0..120).map(|i| json!(i)).collect();
+ let crusher = SmartCrusher::new(SmartCrusherConfig::default());
+
+ let kept = crush_array_values(&crusher, &items);
+ // `kept` includes the trailing sentinel, so the crusher's drop
+ // count is original minus the data items (all but the last).
+ let dropped = items.len() - (kept.len() - 1);
+ assert!(
+ kept.len() < items.len(),
+ "expected drops for a 120-item number array"
+ );
+ let sentinel = kept
+ .last()
+ .and_then(|v| v.as_str())
+ .expect("last element is the string sentinel");
+ assert!(sentinel.starts_with(&format!("… {dropped} more items ")));
+ let stored = stored_original(&crusher, sentinel);
+ assert_eq!(stored.len(), 120);
+ }
+
+ #[test]
+ fn mixed_array_drops_emit_single_sentinel() {
+ let items: Vec<Value> = (0..120)
+ .map(|i| {
+ if i % 2 == 0 {
+ json!(format!("entry-{i}"))
+ } else {
+ json!(i)
+ }
+ })
+ .collect();
+ let crusher = SmartCrusher::new(SmartCrusherConfig::default());
+
+ let kept = crush_array_values(&crusher, &items);
+ // `kept` includes the trailing sentinel, so the crusher's drop
+ // count is original minus the data items (all but the last).
+ let dropped = items.len() - (kept.len() - 1);
+ assert!(
+ kept.len() < items.len(),
+ "expected drops for a 120-item mixed array"
+ );
+ let sentinels: Vec<&str> = kept
+ .iter()
+ .filter_map(|v| v.as_str())
+ .filter(|s| s.contains("_items_offloaded>>"))
+ .collect();
+ assert_eq!(
+ sentinels.len(),
+ 1,
+ "exactly one sentinel, got {sentinels:?}"
+ );
+ assert!(sentinels[0].starts_with(&format!("… {dropped} more items ")));
+ let stored = stored_original(&crusher, sentinels[0]);
+ assert_eq!(stored.len(), 120);
+ }
+
#[test]
fn default_crush_ignores_opt_in_prose_hook() {
let crusher = SmartCrusher::new(SmartCrusherConfig::default());
diff --git a/headroom/transforms/smart_crusher.py b/headroom/transforms/smart_crusher.py
index 241a0013e..c7a0b0917 100644
--- a/headroom/transforms/smart_crusher.py
+++ b/headroom/transforms/smart_crusher.py
@@ -79,18 +79,43 @@ _SUPPORTED_COMPACTION_FORMATS = ("csv-schema", "json", "markdown-kv")
CCR_SENTINEL_KEY = "_ccr_dropped"
+# Prefix/infix identifying the scalar-array drop sentinel emitted by the
+# Rust crusher when string/number/mixed array items are dropped with
+# `enable_ccr_marker` on (headroomlabs-ai/headroom#3650). Unlike the
+# dict sentinel object above, this one is a plain string so string
+# arrays keep a uniform element type.
+SCALAR_SENTINEL_PREFIX = "… "
+SCALAR_SENTINEL_INFIX = " more items "
+
+
def is_ccr_sentinel(item: Any) -> bool:
- """True if `item` is a CCR-dropped sentinel object."""
- return isinstance(item, dict) and CCR_SENTINEL_KEY in item
+ """True if `item` is a CCR-dropped sentinel.
+
+ Covers both sentinel shapes: the dict object (``{"_ccr_dropped": ...}``)
+ emitted for dict arrays, and the plain-string sentinel
+ (``"… N more items <<ccr:HASH N_items_offloaded>>"``) appended to
+ string/number/mixed arrays when items are dropped
+ (headroomlabs-ai/headroom#3650).
+ """
+ if isinstance(item, dict):
+ return CCR_SENTINEL_KEY in item
+ return (
+ isinstance(item, str)
+ and item.startswith(SCALAR_SENTINEL_PREFIX)
+ and SCALAR_SENTINEL_INFIX in item
+ and "<<ccr:" in item
+ )
def strip_ccr_sentinels(items: Any) -> Any:
"""Return `items` with any CCR-dropped sentinel objects filtered out.
Pass this through any iteration over a compressed array's contents
- when your code expects a uniform-schema list of records. The sentinel
- carries a `<<ccr:HASH ...>>` marker for the LLM and shouldn't be
- confused for a record — it has only the `_ccr_dropped` key.
+ when your code expects a uniform-schema list of records. The dict
+ sentinel carries a `<<ccr:HASH ...>>` marker for the LLM and shouldn't
+ be confused for a record — it has only the `_ccr_dropped` key. The
+ string sentinel (scalar arrays) likewise marks dropped items without
+ pretending to be data.
Non-list inputs pass through unchanged so callers can wrap whatever
`json.loads` returned without first checking the shape.
diff --git a/tests/parity/fixtures/smart_crusher/mixed_array_3f929019b62f.json b/tests/parity/fixtures/smart_crusher/mixed_array_3f929019b62f.json
index 4a9f61509..99eef973e 100644
--- a/tests/parity/fixtures/smart_crusher/mixed_array_3f929019b62f.json
+++ b/tests/parity/fixtures/smart_crusher/mixed_array_3f929019b62f.json
@@ -24,7 +24,7 @@
"input_sha256": "3f929019b62fa97b372e2d8aad705dbef029ddb7934acdb237083fdead8162c0",
"label": "mixed_array",
"output": {
- "compressed": "[\"start\",0,1,2,3,18,19,\"middle\",\"end\",\"end\",\"end\",\"end\",\"end\"]",
+ "compressed": "[\"start\",0,1,2,3,18,19,\"middle\",\"end\",\"end\",\"end\",\"end\",\"end\",\"\u2026 14 more items <<ccr:c698823f3fa0 14_items_offloaded>>\"]",
"original": "[\"start\", 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, \"middle\", \"end\", \"end\", \"end\", \"end\", \"end\"]",
"strategy": "mixed:adaptive(27->13,str:7->7,num:20)(27->13)",
"was_modified": true
diff --git a/tests/parity/fixtures/smart_crusher/number_array_40_changepoint_8cd416c24d53.json b/tests/parity/fixtures/smart_crusher/number_array_40_changepoint_8cd416c24d53.json
index 2514bdf51..2147c4929 100644
--- a/tests/parity/fixtures/smart_crusher/number_array_40_changepoint_8cd416c24d53.json
+++ b/tests/parity/fixtures/smart_crusher/number_array_40_changepoint_8cd416c24d53.json
@@ -24,7 +24,7 @@
"input_sha256": "8cd416c24d53a5c61ebf4ad2e06963169de552cf163331df276773459af00275",
"label": "number_array_40_changepoint",
"output": {
- "compressed": "[10,11,12,10,10,10,10,10,10,51,54,57,60,68,69]",
+ "compressed": "[10,11,12,10,10,10,10,10,10,51,54,57,60,68,69,\"\u2026 25 more items <<ccr:905235220a7b 25_items_offloaded>>\"]",
"original": "[10, 11, 12, 10, 11, 12, 10, 11, 12, 10, 11, 12, 10, 11, 12, 10, 11, 12, 10, 11, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69]",
"strategy": "number:adaptive(40->15,min=10,max=69,mean=35.23,median=31,stddev=24.94,p25=11,p75=59.25)(40->15)",
"was_modified": true
diff --git a/tests/parity/fixtures/smart_crusher/string_array_25_6fe4a307de81.json b/tests/parity/fixtures/smart_crusher/string_array_25_6fe4a307de81.json
index 58f8122a2..8c246b431 100644
--- a/tests/parity/fixtures/smart_crusher/string_array_25_6fe4a307de81.json
+++ b/tests/parity/fixtures/smart_crusher/string_array_25_6fe4a307de81.json
@@ -24,7 +24,7 @@
"input_sha256": "6fe4a307de817df20507df7fc88f0237c203a785c550696abb1f6b8dd413b40e",
"label": "string_array_25",
"output": {
- "compressed": "[\"event 0: something happened at index 0\",\"event 1: something happened at index 1\",\"event 2: something happened at index 2\",\"event 3: something happened at index 3\",\"event 4: something happened at index 4\",\"event 6: something happened at index 6\",\"event 8: something happened at index 8\",\"event 10: something happened at index 10\",\"event 12: something happened at index 12\",\"event 14: something happened at index 14\",\"event 16: something happened at index 16\",\"event 18: something happened at index 18\",\"event 20: something happened at index 20\",\"event 23: something happened at index 23\",\"event 24: something happened at index 24\"]",
+ "compressed": "[\"event 0: something happened at index 0\",\"event 1: something happened at index 1\",\"event 2: something happened at index 2\",\"event 3: something happened at index 3\",\"event 4: something happened at index 4\",\"event 6: something happened at index 6\",\"event 8: something happened at index 8\",\"event 10: something happened at index 10\",\"event 12: something happened at index 12\",\"event 14: something happened at index 14\",\"event 16: something happened at index 16\",\"event 18: something happened at index 18\",\"event 20: something happened at index 20\",\"event 23: something happened at index 23\",\"event 24: something happened at index 24\",\"\u2026 10 more items <<ccr:9b64aca4dbf5 10_items_offloaded>>\"]",
"original": "[\"event 0: something happened at index 0\", \"event 1: something happened at index 1\", \"event 2: something happened at index 2\", \"event 3: something happened at index 3\", \"event 4: something happened at index 4\", \"event 5: something happened at index 5\", \"event 6: something happened at index 6\", \"event 7: something happened at index 7\", \"event 8: something happened at index 8\", \"event 9: something happened at index 9\", \"event 10: something happened at index 10\", \"event 11: something happened at index 11\", \"event 12: something happened at index 12\", \"event 13: something happened at index 13\", \"event 14: something happened at index 14\", \"event 15: something happened at index 15\", \"event 16: something happened at index 16\", \"event 17: something happened at index 17\", \"event 18: something happened at index 18\", \"event 19: something happened at index 19\", \"event 20: something happened at index 20\", \"event 21: something happened at index 21\", \"event 22: something happened at index 22\", \"event 23: something happened at index 23\", \"event 24: something happened at index 24\"]",
"strategy": "string:adaptive(25->15)(25->15)",
"was_modified": true
diff --git a/tests/test_transforms/test_smart_crusher_issue_3650.py b/tests/test_transforms/test_smart_crusher_issue_3650.py
new file mode 100644
index 000000000..ea016bf82
--- /dev/null
+++ b/tests/test_transforms/test_smart_crusher_issue_3650.py
@@ -0,0 +1,79 @@
+"""Regression tests for headroomlabs-ai/headroom#3650.
+
+Scalar (string/number/mixed) arrays used to drop items silently: the
+crusher returned only the retained items with no marker, so e.g. 120
+slugs came back as 16 with no trace of the other 104. Dropped scalar
+items now get a visible, retrievable CCR sentinel appended to the
+array.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+
+import pytest
+
+from headroom.transforms.content_router import ContentRouter, ContentRouterConfig
+from headroom.transforms.smart_crusher import (
+ is_ccr_sentinel,
+ strip_ccr_sentinels,
+)
+
+_CCR_RE = re.compile(r"<<ccr:([0-9a-f]{12}) ")
+
+
+def _issue_payload() -> str:
+ # The exact shape from the issue report.
+ return json.dumps({"slugs": [f"r{i}" for i in range(120)], "total": 120})
+
+
+def test_scalar_sentinel_recognized_by_python_shim() -> None:
+ """Pure-Python coverage: the new string sentinel is a CCR sentinel."""
+ sentinel = "… 104 more items <<ccr:4f03caae30bb 104_items_offloaded>>"
+ assert is_ccr_sentinel(sentinel)
+ # The dict sentinel keeps working.
+ assert is_ccr_sentinel({"_ccr_dropped": "<<ccr:abc123 5_rows_offloaded>>"})
+ # Real data is untouched.
+ assert not is_ccr_sentinel("r42")
+ assert not is_ccr_sentinel({"slugs": ["r1"]})
+ assert not is_ccr_sentinel(42)
+
+ items = ["r0", sentinel, "r1", {"_ccr_dropped": "<<ccr:abc123 1_rows_offloaded>>"}]
+ assert strip_ccr_sentinels(items) == ["r0", "r1"]
+ assert strip_ccr_sentinels("not-a-list") == "not-a-list"
+
+
+def test_issue_3650_router_emits_visible_retrievable_sentinel() -> None:
+ """End-to-end: the issue's payload truncates visibly and retrievably."""
+ pytest.importorskip("headroom._core") # needs the native extension
+ router = ContentRouter(ContentRouterConfig())
+ result = router.compress(_issue_payload())
+ compressed = result.compressed
+
+ # 1. The truncation is visible: a sentinel names the drop count and
+ # carries a CCR marker. (Before the fix, 104 slugs vanished with
+ # no marker at all.)
+ assert "more items" in compressed, compressed[:300]
+ m = _CCR_RE.search(compressed)
+ assert m, f"no CCR marker in compressed output: {compressed[:300]}"
+ ccr_hash = m.group(1)
+
+ # 2. The marker retrieves the FULL original array: the Python
+ # SmartCrusher mirrors Rust CCR markers into the process
+ # compression store.
+ from headroom.cache.compression_store import get_compression_store
+
+ entry = get_compression_store().retrieve(ccr_hash)
+ assert entry is not None, "CCR marker hash must resolve in the compression store"
+ original = json.loads(entry.original_content)
+ assert len(original) == 120
+ assert original[0] == "r0"
+ assert original[119] == "r119"
+
+ # 3. The sibling field is untouched and the output still parses.
+ doc = json.loads(compressed)
+ assert doc["total"] == 120
+ assert isinstance(doc["slugs"], list)
+ assert len(doc["slugs"]) < 120 # some were dropped…
+ assert any(is_ccr_sentinel(x) for x in doc["slugs"]) # …but marked