Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(smart-crusher): disclose dropped scalar-array items with a retrievable ccr sentinel (#3692)
Raphael Malikianc1fc84ade6c8
diff --git a/crates/headroom-core/src/transforms/smart_crusher/crusher.rs b/crates/headroom-core/src/transforms/smart_crusher/crusher.rsindex 0dc34df77..9fb40c9e7 100644--- a/crates/headroom-core/src/transforms/smart_crusher/crusher.rs+++ b/crates/headroom-core/src/transforms/smart_crusher/crusher.rs@@ -595,20 +595,31 @@ impl SmartCrusher {let strs: Vec<&str> = arr.iter().filter_map(|v| v.as_str()).collect();let (crushed, strategy) = crush_string_array(&strs, &self.config, bias);info_parts.push(format!("{}({}->{})", strategy, n, crushed.len()));- let crushed_values: Vec<Value> =+ let mut crushed_values: Vec<Value> =crushed.into_iter().map(Value::String).collect();+ // #3650: dropped strings get a retrievable+ // sentinel — silent truncation is a data-loss+ // bug even when the marker gate is on.+ self.append_scalar_drop_sentinel(&mut crushed_values, arr);return (Value::Array(crushed_values), info_parts.join(","));}ArrayType::NumberArray => {let (crushed, strategy) = crush_number_array(arr, &self.config, bias);info_parts.push(format!("{}({}->{})", strategy, n, crushed.len()));- return (Value::Array(crushed), info_parts.join(","));+ let mut crushed_values = crushed;+ // #3650: same sentinel contract as StringArray.+ self.append_scalar_drop_sentinel(&mut crushed_values, arr);+ return (Value::Array(crushed_values), info_parts.join(","));}ArrayType::MixedArray => {let (crushed, strategy) =self.crush_mixed_array(arr, query_context, bias);info_parts.push(format!("{}({}->{})", strategy, n, crushed.len()));- return (Value::Array(crushed), info_parts.join(","));+ let mut crushed_values = crushed;+ // #3650: same sentinel contract as the+ // StringArray/NumberArray arms.+ self.append_scalar_drop_sentinel(&mut crushed_values, arr);+ return (Value::Array(crushed_values), info_parts.join(","));}// NestedArray, BoolArray, Empty → fall through// to recursive descent.@@ -1107,6 +1118,30 @@ impl SmartCrusher {);(result, strategy)}++ /// Appends a visible, retrievable CCR sentinel to `kept` when items were+ /// dropped from `original` and the marker gate is on.+ ///+ /// Mirrors the dict-array path: the full original array is serialized+ /// once (`canonical_array_json`), hashed with the shared+ /// `hash_canonical` (SHA256, first 12 hex chars — the same scheme the+ /// Python bridge and retrieval use), and stored in the CCR store so+ /// the marker round-trips through retrieval. No-op when nothing was+ /// dropped or `enable_ccr_marker` is false.+ fn append_scalar_drop_sentinel(&self, kept: &mut Vec<Value>, original: &[Value]) {+ let dropped = original.len().saturating_sub(kept.len());+ if dropped == 0 || !self.config.enable_ccr_marker {+ return;+ }+ let canonical = canonical_array_json(original);+ let hash = hash_canonical(&canonical);+ if let Some(store) = &self.ccr_store {+ store.put(&hash, &canonical);+ }+ kept.push(Value::String(format!(+ "… {dropped} more items <<ccr:{hash} {dropped}_items_offloaded>>"+ )));+ }}// ---------- helpers ----------@@ -1275,6 +1310,166 @@ mod tests {use super::*;use serde_json::json;+ /// Extracts the CCR hash from a scalar drop sentinel produced by+ /// `append_scalar_drop_sentinel`.+ fn scalar_sentinel_hash(s: &str) -> &str {+ let start = s.find("<<ccr:").expect("sentinel must carry a CCR marker") + 6;+ let end = s[start..].find(' ').expect("marker must have a tag") + start;+ &s[start..end]+ }++ /// Crushes a JSON array string and returns the parsed compressed array.+ fn crush_array_values(crusher: &SmartCrusher, items: &[Value]) -> Vec<Value> {+ let out = crusher.crush(&serde_json::to_string(items).unwrap(), "", 1.0);+ serde_json::from_str::<Value>(&out.compressed)+ .expect("compressed output is JSON")+ .as_array()+ .expect("output stays an array")+ .clone()+ }++ /// Looks up the stored original array for a sentinel's CCR hash.+ fn stored_original(crusher: &SmartCrusher, sentinel: &str) -> Vec<Value> {+ let hash = scalar_sentinel_hash(sentinel);+ let raw = crusher+ .ccr_store()+ .and_then(|store| store.get(hash))+ .expect("sentinel hash must be in the CCR store");+ serde_json::from_str(&raw).expect("stored payload is a JSON array")+ }++ // Regression tests for headroomlabs-ai/headroom#3650: scalar+ // (string/number/mixed) array crushers dropped items silently. They+ // must now emit a visible, retrievable CCR sentinel.++ #[test]+ fn string_array_drops_emit_retrievable_sentinel() {+ // The issue's repro shape: 120 short slugs.+ let items: Vec<Value> = (0..120).map(|i| json!(format!("r{i}"))).collect();+ let config = SmartCrusherConfig {+ enable_ccr_marker: true,+ ..Default::default()+ };+ let crusher = SmartCrusher::new(config);++ let kept = crush_array_values(&crusher, &items);+ // `kept` includes the trailing sentinel, so the crusher's drop+ // count is original minus the data items (all but the last).+ let dropped = items.len() - (kept.len() - 1);+ assert!(+ kept.len() < items.len(),+ "expected drops for a 120-item string array"+ );+ let sentinel = kept+ .last()+ .and_then(|v| v.as_str())+ .expect("last element is the string sentinel");+ assert!(+ sentinel.starts_with(&format!("… {dropped} more items ")),+ "sentinel shows the visible drop count, got: {sentinel}"+ );+ assert!(+ sentinel.contains("<<ccr:")+ && sentinel.ends_with(&format!("{dropped}_items_offloaded>>")),+ "sentinel carries a CCR marker, got: {sentinel}"+ );+ // The marker hash must retrieve the FULL original array.+ let stored = stored_original(&crusher, sentinel);+ assert_eq!(stored.len(), 120);+ assert_eq!(stored[119], json!("r119"));+ }++ #[test]+ fn string_array_sentinel_suppressed_when_marker_gate_off() {+ let items: Vec<Value> = (0..120).map(|i| json!(format!("r{i}"))).collect();+ let config = SmartCrusherConfig {+ enable_ccr_marker: false,+ ..Default::default()+ };+ let crusher = SmartCrusher::new(config);++ let kept = crush_array_values(&crusher, &items);+ assert!(+ !kept+ .iter()+ .any(|v| v.as_str().is_some_and(|s| s.contains("<<ccr:"))),+ "no sentinel when the marker gate is off"+ );+ }++ #[test]+ fn string_array_no_sentinel_when_nothing_dropped() {+ let items: Vec<Value> = (0..5).map(|i| json!(format!("r{i}"))).collect();+ let crusher = SmartCrusher::new(SmartCrusherConfig::default());++ let kept = crush_array_values(&crusher, &items);+ assert_eq!(kept.len(), items.len());+ assert!(+ !kept+ .iter()+ .any(|v| v.as_str().is_some_and(|s| s.contains("<<ccr:"))),+ "no sentinel when nothing was dropped"+ );+ }++ #[test]+ fn number_array_drops_emit_retrievable_sentinel() {+ let items: Vec<Value> = (0..120).map(|i| json!(i)).collect();+ let crusher = SmartCrusher::new(SmartCrusherConfig::default());++ let kept = crush_array_values(&crusher, &items);+ // `kept` includes the trailing sentinel, so the crusher's drop+ // count is original minus the data items (all but the last).+ let dropped = items.len() - (kept.len() - 1);+ assert!(+ kept.len() < items.len(),+ "expected drops for a 120-item number array"+ );+ let sentinel = kept+ .last()+ .and_then(|v| v.as_str())+ .expect("last element is the string sentinel");+ assert!(sentinel.starts_with(&format!("… {dropped} more items ")));+ let stored = stored_original(&crusher, sentinel);+ assert_eq!(stored.len(), 120);+ }++ #[test]+ fn mixed_array_drops_emit_single_sentinel() {+ let items: Vec<Value> = (0..120)+ .map(|i| {+ if i % 2 == 0 {+ json!(format!("entry-{i}"))+ } else {+ json!(i)+ }+ })+ .collect();+ let crusher = SmartCrusher::new(SmartCrusherConfig::default());++ let kept = crush_array_values(&crusher, &items);+ // `kept` includes the trailing sentinel, so the crusher's drop+ // count is original minus the data items (all but the last).+ let dropped = items.len() - (kept.len() - 1);+ assert!(+ kept.len() < items.len(),+ "expected drops for a 120-item mixed array"+ );+ let sentinels: Vec<&str> = kept+ .iter()+ .filter_map(|v| v.as_str())+ .filter(|s| s.contains("_items_offloaded>>"))+ .collect();+ assert_eq!(+ sentinels.len(),+ 1,+ "exactly one sentinel, got {sentinels:?}"+ );+ assert!(sentinels[0].starts_with(&format!("… {dropped} more items ")));+ let stored = stored_original(&crusher, sentinels[0]);+ assert_eq!(stored.len(), 120);+ }+#[test]fn default_crush_ignores_opt_in_prose_hook() {let crusher = SmartCrusher::new(SmartCrusherConfig::default());diff --git a/headroom/transforms/smart_crusher.py b/headroom/transforms/smart_crusher.pyindex 241a0013e..c7a0b0917 100644--- a/headroom/transforms/smart_crusher.py+++ b/headroom/transforms/smart_crusher.py@@ -79,18 +79,43 @@ _SUPPORTED_COMPACTION_FORMATS = ("csv-schema", "json", "markdown-kv")CCR_SENTINEL_KEY = "_ccr_dropped"+# Prefix/infix identifying the scalar-array drop sentinel emitted by the+# Rust crusher when string/number/mixed array items are dropped with+# `enable_ccr_marker` on (headroomlabs-ai/headroom#3650). Unlike the+# dict sentinel object above, this one is a plain string so string+# arrays keep a uniform element type.+SCALAR_SENTINEL_PREFIX = "… "+SCALAR_SENTINEL_INFIX = " more items "++def is_ccr_sentinel(item: Any) -> bool:- """True if `item` is a CCR-dropped sentinel object."""- return isinstance(item, dict) and CCR_SENTINEL_KEY in item+ """True if `item` is a CCR-dropped sentinel.++ Covers both sentinel shapes: the dict object (``{"_ccr_dropped": ...}``)+ emitted for dict arrays, and the plain-string sentinel+ (``"… N more items <<ccr:HASH N_items_offloaded>>"``) appended to+ string/number/mixed arrays when items are dropped+ (headroomlabs-ai/headroom#3650).+ """+ if isinstance(item, dict):+ return CCR_SENTINEL_KEY in item+ return (+ isinstance(item, str)+ and item.startswith(SCALAR_SENTINEL_PREFIX)+ and SCALAR_SENTINEL_INFIX in item+ and "<<ccr:" in item+ )def strip_ccr_sentinels(items: Any) -> Any:"""Return `items` with any CCR-dropped sentinel objects filtered out.Pass this through any iteration over a compressed array's contents- when your code expects a uniform-schema list of records. The sentinel- carries a `<<ccr:HASH ...>>` marker for the LLM and shouldn't be- confused for a record — it has only the `_ccr_dropped` key.+ when your code expects a uniform-schema list of records. The dict+ sentinel carries a `<<ccr:HASH ...>>` marker for the LLM and shouldn't+ be confused for a record — it has only the `_ccr_dropped` key. The+ string sentinel (scalar arrays) likewise marks dropped items without+ pretending to be data.Non-list inputs pass through unchanged so callers can wrap whatever`json.loads` returned without first checking the shape.diff --git a/tests/parity/fixtures/smart_crusher/mixed_array_3f929019b62f.json b/tests/parity/fixtures/smart_crusher/mixed_array_3f929019b62f.jsonindex 4a9f61509..99eef973e 100644--- a/tests/parity/fixtures/smart_crusher/mixed_array_3f929019b62f.json+++ b/tests/parity/fixtures/smart_crusher/mixed_array_3f929019b62f.json@@ -24,7 +24,7 @@"input_sha256": "3f929019b62fa97b372e2d8aad705dbef029ddb7934acdb237083fdead8162c0","label": "mixed_array","output": {- "compressed": "[\"start\",0,1,2,3,18,19,\"middle\",\"end\",\"end\",\"end\",\"end\",\"end\"]",+ "compressed": "[\"start\",0,1,2,3,18,19,\"middle\",\"end\",\"end\",\"end\",\"end\",\"end\",\"\u2026 14 more items <<ccr:c698823f3fa0 14_items_offloaded>>\"]","original": "[\"start\", 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, \"middle\", \"end\", \"end\", \"end\", \"end\", \"end\"]","strategy": "mixed:adaptive(27->13,str:7->7,num:20)(27->13)","was_modified": truediff --git a/tests/parity/fixtures/smart_crusher/number_array_40_changepoint_8cd416c24d53.json b/tests/parity/fixtures/smart_crusher/number_array_40_changepoint_8cd416c24d53.jsonindex 2514bdf51..2147c4929 100644--- a/tests/parity/fixtures/smart_crusher/number_array_40_changepoint_8cd416c24d53.json+++ b/tests/parity/fixtures/smart_crusher/number_array_40_changepoint_8cd416c24d53.json@@ -24,7 +24,7 @@"input_sha256": "8cd416c24d53a5c61ebf4ad2e06963169de552cf163331df276773459af00275","label": "number_array_40_changepoint","output": {- "compressed": "[10,11,12,10,10,10,10,10,10,51,54,57,60,68,69]",+ "compressed": "[10,11,12,10,10,10,10,10,10,51,54,57,60,68,69,\"\u2026 25 more items <<ccr:905235220a7b 25_items_offloaded>>\"]","original": "[10, 11, 12, 10, 11, 12, 10, 11, 12, 10, 11, 12, 10, 11, 12, 10, 11, 12, 10, 11, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69]","strategy": "number:adaptive(40->15,min=10,max=69,mean=35.23,median=31,stddev=24.94,p25=11,p75=59.25)(40->15)","was_modified": truediff --git a/tests/parity/fixtures/smart_crusher/string_array_25_6fe4a307de81.json b/tests/parity/fixtures/smart_crusher/string_array_25_6fe4a307de81.jsonindex 58f8122a2..8c246b431 100644--- a/tests/parity/fixtures/smart_crusher/string_array_25_6fe4a307de81.json+++ b/tests/parity/fixtures/smart_crusher/string_array_25_6fe4a307de81.json@@ -24,7 +24,7 @@"input_sha256": "6fe4a307de817df20507df7fc88f0237c203a785c550696abb1f6b8dd413b40e","label": "string_array_25","output": {- "compressed": "[\"event 0: something happened at index 0\",\"event 1: something happened at index 1\",\"event 2: something happened at index 2\",\"event 3: something happened at index 3\",\"event 4: something happened at index 4\",\"event 6: something happened at index 6\",\"event 8: something happened at index 8\",\"event 10: something happened at index 10\",\"event 12: something happened at index 12\",\"event 14: something happened at index 14\",\"event 16: something happened at index 16\",\"event 18: something happened at index 18\",\"event 20: something happened at index 20\",\"event 23: something happened at index 23\",\"event 24: something happened at index 24\"]",+ "compressed": "[\"event 0: something happened at index 0\",\"event 1: something happened at index 1\",\"event 2: something happened at index 2\",\"event 3: something happened at index 3\",\"event 4: something happened at index 4\",\"event 6: something happened at index 6\",\"event 8: something happened at index 8\",\"event 10: something happened at index 10\",\"event 12: something happened at index 12\",\"event 14: something happened at index 14\",\"event 16: something happened at index 16\",\"event 18: something happened at index 18\",\"event 20: something happened at index 20\",\"event 23: something happened at index 23\",\"event 24: something happened at index 24\",\"\u2026 10 more items <<ccr:9b64aca4dbf5 10_items_offloaded>>\"]","original": "[\"event 0: something happened at index 0\", \"event 1: something happened at index 1\", \"event 2: something happened at index 2\", \"event 3: something happened at index 3\", \"event 4: something happened at index 4\", \"event 5: something happened at index 5\", \"event 6: something happened at index 6\", \"event 7: something happened at index 7\", \"event 8: something happened at index 8\", \"event 9: something happened at index 9\", \"event 10: something happened at index 10\", \"event 11: something happened at index 11\", \"event 12: something happened at index 12\", \"event 13: something happened at index 13\", \"event 14: something happened at index 14\", \"event 15: something happened at index 15\", \"event 16: something happened at index 16\", \"event 17: something happened at index 17\", \"event 18: something happened at index 18\", \"event 19: something happened at index 19\", \"event 20: something happened at index 20\", \"event 21: something happened at index 21\", \"event 22: something happened at index 22\", \"event 23: something happened at index 23\", \"event 24: something happened at index 24\"]","strategy": "string:adaptive(25->15)(25->15)","was_modified": truediff --git a/tests/test_transforms/test_smart_crusher_issue_3650.py b/tests/test_transforms/test_smart_crusher_issue_3650.pynew file mode 100644index 000000000..ea016bf82--- /dev/null+++ b/tests/test_transforms/test_smart_crusher_issue_3650.py@@ -0,0 +1,79 @@+"""Regression tests for headroomlabs-ai/headroom#3650.++Scalar (string/number/mixed) arrays used to drop items silently: the+crusher returned only the retained items with no marker, so e.g. 120+slugs came back as 16 with no trace of the other 104. Dropped scalar+items now get a visible, retrievable CCR sentinel appended to the+array.+"""++from __future__ import annotations++import json+import re++import pytest++from headroom.transforms.content_router import ContentRouter, ContentRouterConfig+from headroom.transforms.smart_crusher import (+ is_ccr_sentinel,+ strip_ccr_sentinels,+)++_CCR_RE = re.compile(r"<<ccr:([0-9a-f]{12}) ")+++def _issue_payload() -> str:+ # The exact shape from the issue report.+ return json.dumps({"slugs": [f"r{i}" for i in range(120)], "total": 120})+++def test_scalar_sentinel_recognized_by_python_shim() -> None:+ """Pure-Python coverage: the new string sentinel is a CCR sentinel."""+ sentinel = "… 104 more items <<ccr:4f03caae30bb 104_items_offloaded>>"+ assert is_ccr_sentinel(sentinel)+ # The dict sentinel keeps working.+ assert is_ccr_sentinel({"_ccr_dropped": "<<ccr:abc123 5_rows_offloaded>>"})+ # Real data is untouched.+ assert not is_ccr_sentinel("r42")+ assert not is_ccr_sentinel({"slugs": ["r1"]})+ assert not is_ccr_sentinel(42)++ items = ["r0", sentinel, "r1", {"_ccr_dropped": "<<ccr:abc123 1_rows_offloaded>>"}]+ assert strip_ccr_sentinels(items) == ["r0", "r1"]+ assert strip_ccr_sentinels("not-a-list") == "not-a-list"+++def test_issue_3650_router_emits_visible_retrievable_sentinel() -> None:+ """End-to-end: the issue's payload truncates visibly and retrievably."""+ pytest.importorskip("headroom._core") # needs the native extension+ router = ContentRouter(ContentRouterConfig())+ result = router.compress(_issue_payload())+ compressed = result.compressed++ # 1. The truncation is visible: a sentinel names the drop count and+ # carries a CCR marker. (Before the fix, 104 slugs vanished with+ # no marker at all.)+ assert "more items" in compressed, compressed[:300]+ m = _CCR_RE.search(compressed)+ assert m, f"no CCR marker in compressed output: {compressed[:300]}"+ ccr_hash = m.group(1)++ # 2. The marker retrieves the FULL original array: the Python+ # SmartCrusher mirrors Rust CCR markers into the process+ # compression store.+ from headroom.cache.compression_store import get_compression_store++ entry = get_compression_store().retrieve(ccr_hash)+ assert entry is not None, "CCR marker hash must resolve in the compression store"+ original = json.loads(entry.original_content)+ assert len(original) == 120+ assert original[0] == "r0"+ assert original[119] == "r119"++ # 3. The sibling field is untouched and the output still parses.+ doc = json.loads(compressed)+ assert doc["total"] == 120+ assert isinstance(doc["slugs"], list)+ assert len(doc["slugs"]) < 120 # some were dropped…+ assert any(is_ccr_sentinel(x) for x in doc["slugs"]) # …but marked