Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix: preserve JSON object fields in SmartCrusher (#3648)

Matt Van Horn545f54414703
diff --git a/crates/headroom-core/src/transforms/smart_crusher/crusher.rs b/crates/headroom-core/src/transforms/smart_crusher/crusher.rs
index a247890a2..9d71197ec 100644
--- a/crates/headroom-core/src/transforms/smart_crusher/crusher.rs
+++ b/crates/headroom-core/src/transforms/smart_crusher/crusher.rs
@@ -446,10 +446,26 @@ impl SmartCrusher {
let (crushed, info) =
self.process_value_with_hook(&parsed, 0, query_context, bias, prose_hook);
+ // When an object and its descendants were left unchanged, keep
+ // the original bytes. Re-serializing would rewrite Unicode
+ // escapes and numeric lexical forms, then advertise a false
+ // compression. Nested skip/compaction still serializes below
+ // because those paths populate `info`. Spaced JSON without a
+ // nested strategy is compact-serialized for historical crush()
+ // / parity fixtures.
+ if crushed == parsed
+ && info.is_empty()
+ && matches!(parsed, Value::Object(_))
+ && !has_json_insignificant_whitespace(content)
+ {
+ return (content.to_string(), false, String::new());
+ }
+
// Re-serialize with Python `safe_json_dumps` formatting:
// compact `(",", ":")` separators + `ensure_ascii=False`,
// preserving object-key insertion order. Matches the Python
- // SmartCrusher output bytes the proxy writes.
+ // SmartCrusher output bytes the proxy writes. When descendants
+ // actually change, this path keeps strategy attribution.
let result = crate::transforms::anchor_selector::python_safe_json_dumps(&crushed);
let was_modified = result != content.trim();
(result, was_modified, info)
@@ -629,8 +645,11 @@ impl SmartCrusher {
}
}
- // Second pass: if the object itself has many keys,
- // compress at the key level.
+ // Second pass: crush_object preserves every property
+ // (object fields are not interchangeable sampled
+ // records). Nested arrays were already compacted above,
+ // so large-array compaction and CCR row offload still
+ // run without deleting their enclosing keys.
if processed.len() >= self.config.min_items_to_analyze {
let (crushed_dict, strategy) = crush_object(&processed, &self.config, bias);
if strategy != "object:passthrough" {
@@ -1213,6 +1232,36 @@ fn hash_canonical(canonical: &str) -> String {
// here because `process_string`'s `string_ccr:<kind>` strategy-info
// label is local to this module's debug-string convention.
+/// True when `s` has JSON whitespace outside of string literals
+/// (spaces after `:` / `,`, pretty-print newlines, etc.). Used to
+/// keep compact unchanged objects on the original-bytes path while
+/// still compact-serializing spaced inputs for historical crush()
+/// output.
+fn has_json_insignificant_whitespace(s: &str) -> bool {
+ let mut in_string = false;
+ let mut escape = false;
+ for ch in s.trim().chars() {
+ if in_string {
+ if escape {
+ escape = false;
+ } else if ch == '\\' {
+ escape = true;
+ } else if ch == '"' {
+ in_string = false;
+ }
+ continue;
+ }
+ if ch == '"' {
+ in_string = true;
+ continue;
+ }
+ if ch.is_whitespace() {
+ return true;
+ }
+ }
+ false
+}
+
fn opaque_kind_label(kind: &super::compaction::OpaqueKind) -> &str {
use super::compaction::OpaqueKind;
match kind {
@@ -1533,18 +1582,70 @@ mod tests {
}
#[test]
- fn crush_serializes_with_python_safe_format() {
+ fn crush_serializes_changed_objects_with_python_safe_format() {
let c = crusher();
- // SmartCrusher uses Python's `safe_json_dumps`: compact
- // separators `(",", ":")` + `ensure_ascii=False`, preserving
- // object-key insertion order. A spaced input round-trips to
- // the compact form.
- let input = r#"{"a": 1, "b": 2, "c": 3}"#;
- let result = c.crush(input, "", 1.0);
- assert_eq!(
- result.compressed, r#"{"a":1,"b":2,"c":3}"#,
- "safe_json_dumps emits compact `,` / `:` separators"
+ // When descendants change, SmartCrusher re-serializes with
+ // Python's `safe_json_dumps`: compact separators `(",", ":")`
+ // + `ensure_ascii=False`. Unchanged objects keep original
+ // bytes instead of being rewritten.
+ let mut inner = String::from("[");
+ for i in 0..30 {
+ if i > 0 {
+ inner.push(',');
+ }
+ inner.push_str(r#"{"status": "ok"}"#);
+ }
+ inner.push(']');
+ let input = format!(r#"{{"a": 1, "data": {inner}}}"#);
+ let result = c.crush(&input, "", 1.0);
+ assert!(
+ result.was_modified,
+ "nested compressible array must trigger serialization"
+ );
+ assert!(
+ result.compressed.contains(r#""a":1"#),
+ "safe_json_dumps emits compact `:`; got {}",
+ result.compressed
+ );
+ assert!(
+ !result.compressed.contains(": "),
+ "changed output must not keep spaced separators: {}",
+ result.compressed
+ );
+ assert_ne!(result.strategy, "passthrough");
+ }
+
+ #[test]
+ fn crush_unchanged_object_keeps_original_bytes() {
+ let c = crusher();
+ // Compact input: escaped Unicode and numeric lexical forms must
+ // survive the passthrough branch rather than being reserialized
+ // (`\u00e9` → `é`, `1.0` → `1`).
+ let compact = r#"{"cafe":"caf\u00e9","n":1.0,"ok":true}"#;
+ let result = c.crush(compact, "", 1.0);
+ assert_eq!(result.compressed, compact);
+ assert!(!result.was_modified);
+ assert_eq!(result.strategy, "passthrough");
+ assert!(
+ !result.compressed.contains("<<ccr:"),
+ "unchanged object must not invent a CCR marker"
);
+
+ // Spaced / pretty-printed unchanged objects still compact-serialize
+ // (historical crush() output, required by parity fixtures) but
+ // must not advertise a key-drop strategy or invent a CCR hash.
+ let spaced = "{ \"cafe\": \"caf\\u00e9\", \"n\": 1.0, \"ok\": true }";
+ let spaced_result = c.crush(spaced, "", 1.0);
+ assert_eq!(spaced_result.strategy, "passthrough");
+ assert!(
+ !spaced_result.strategy.contains("adaptive"),
+ "must not report a key-drop strategy: {}",
+ spaced_result.strategy
+ );
+ assert!(!spaced_result.compressed.contains("<<ccr:"));
+ let parsed: Value = serde_json::from_str(&spaced_result.compressed).unwrap();
+ assert_eq!(parsed["cafe"], "café");
+ assert_eq!(parsed["ok"], true);
}
#[test]
@@ -1552,6 +1653,7 @@ mod tests {
let c = crusher();
// Top-level dict with a nested array of 30 identical items.
// The inner array should compress (low_uniqueness path).
+ // Sibling fields stay even though the array is rewritten.
let mut inner = String::from("[");
for i in 0..30 {
if i > 0 {
@@ -1560,12 +1662,105 @@ mod tests {
inner.push_str(r#"{"status":"ok"}"#);
}
inner.push(']');
- let input = format!(r#"{{"data": {}}}"#, inner);
+ let input =
+ format!(r#"{{"label":"keep","data":{inner},"empty":[],"flag":false,"missing":null}}"#);
let result = c.crush(&input, "", 1.0);
assert!(
result.was_modified,
"nested compressible array must be crushed even inside a wrapper object"
);
+ let parsed: Value = serde_json::from_str(&result.compressed).expect("output is JSON");
+ assert_eq!(parsed["label"], "keep");
+ assert_eq!(parsed["empty"], json!([]));
+ assert_eq!(parsed["flag"], false);
+ assert_eq!(parsed["missing"], Value::Null);
+ assert!(
+ parsed.get("data").is_some(),
+ "enclosing array property must survive"
+ );
+ }
+
+ /// Issue #3634 MCP `saga-mcp_task_get` payload (813 bytes, original
+ /// key order, tags encoded as a JSON string).
+ const MCP_TASK_GET_JSON: &str = r#"{"id":79,"epic_id":9,"title":"test","description":null,"status":"done","priority":"medium","sort_order":0,"assigned_to":null,"estimated_hours":null,"actual_hours":null,"due_date":null,"source_ref":null,"metadata":"{}","created_at":"2026-09-15 20:32:06","updated_at":"2026-09-16 11:59:27","description_locked":0,"is_deleted":0,"deleted_at":null,"deleted_by":null,"delete_reason":null,"epic_name":"latency-routing","tags":"[\"cherry-pick\",\"dedicated branch\"]","subtasks":[{"id":163,"task_id":79,"title":"test2","status":"todo","sort_order":1,"created_at":"2026-09-15 20:32:13","updated_at":"2026-09-16 14:02:26"},{"id":165,"task_id":79,"title":"test4","status":"todo","sort_order":2,"created_at":"2026-09-15 20:38:47","updated_at":"2026-09-17 20:29:04"}],"notes":[],"comments":[],"depends_on":[],"dependents":[]}"#;
+
+ #[test]
+ fn crush_preserves_mcp_task_object_fields() {
+ let c = crusher();
+ let result = c.crush(MCP_TASK_GET_JSON, "", 1.0);
+ assert_eq!(
+ result.compressed, MCP_TASK_GET_JSON,
+ "compact unchanged object must stay byte-identical"
+ );
+ assert!(!result.was_modified);
+ assert_eq!(result.strategy, "passthrough");
+ assert!(
+ !result.strategy.contains("adaptive"),
+ "must not report a key-drop strategy: {}",
+ result.strategy
+ );
+ assert!(
+ !result.compressed.contains("<<ccr:"),
+ "must not invent a CCR marker"
+ );
+ let parsed: Value = serde_json::from_str(&result.compressed).expect("output is JSON");
+ assert_eq!(parsed["tags"], "[\"cherry-pick\",\"dedicated branch\"]");
+ let subtasks = parsed["subtasks"].as_array().expect("subtasks array");
+ assert_eq!(subtasks.len(), 2);
+ assert_eq!(subtasks[0]["title"], "test2");
+ assert_eq!(subtasks[1]["title"], "test4");
+ assert_eq!(parsed["description"], Value::Null);
+ assert_eq!(parsed["notes"], json!([]));
+ }
+
+ #[test]
+ fn nested_array_offload_keeps_enclosing_object_keys() {
+ // Force the lossy row-drop path so CCR offload fires. The
+ // enclosing object must keep every sibling field, and the
+ // stored payload must round-trip as the original rows.
+ let cfg = SmartCrusherConfig {
+ lossless_min_savings_ratio: 0.99,
+ ..SmartCrusherConfig::default()
+ };
+ let c = SmartCrusher::new(cfg);
+ let rows: Vec<Value> = (0..50).map(|_| json!({"status": "ok"})).collect();
+ let doc = json!({
+ "keep_me": "sibling",
+ "rows": rows,
+ "also_keep": null,
+ "flag": true,
+ "empty": []
+ });
+ let content = serde_json::to_string(&doc).unwrap();
+ let result = c.crush(&content, "", 1.0);
+ assert!(result.was_modified, "lossy nested array should compress");
+ assert_ne!(result.strategy, "passthrough");
+ let parsed: Value = serde_json::from_str(&result.compressed).expect("output is JSON");
+ assert_eq!(parsed["keep_me"], "sibling");
+ assert_eq!(parsed["also_keep"], Value::Null);
+ assert_eq!(parsed["flag"], true);
+ assert_eq!(parsed["empty"], json!([]));
+ assert!(
+ parsed.get("rows").is_some(),
+ "enclosing array property must survive"
+ );
+ assert!(
+ result.compressed.contains("<<ccr:"),
+ "offload must emit a CCR marker: {}",
+ result.compressed
+ );
+ let marker_at = result
+ .compressed
+ .find("<<ccr:")
+ .expect("CCR marker present");
+ let hash = &result.compressed[marker_at + 6..marker_at + 18];
+ let stored = c
+ .ccr_store()
+ .expect("default crusher has a store")
+ .get(hash)
+ .expect("offloaded rows are retrievable");
+ let recovered: Value = serde_json::from_str(&stored).expect("stored JSON");
+ assert_eq!(recovered, Value::Array(rows));
}
#[test]
@@ -2134,6 +2329,12 @@ mod tests {
let lossy: Value = serde_json::from_str(&crusher().crush(&content, "", 1.0).compressed)
.expect("default output is JSON");
assert_ne!(lossy, doc, "default config should drop items");
+ // Object keys are not sampled records: default mode may crush
+ // nested arrays but must keep every property of `meta`.
+ assert_eq!(
+ lossy["meta"], doc["meta"],
+ "default mode must keep object keys"
+ );
let strict = SmartCrusher::new(SmartCrusherConfig {
lossless_only: true,
diff --git a/crates/headroom-core/src/transforms/smart_crusher/crushers.rs b/crates/headroom-core/src/transforms/smart_crusher/crushers.rs
index a6f3ecec2..1d203a8fd 100644
--- a/crates/headroom-core/src/transforms/smart_crusher/crushers.rs
+++ b/crates/headroom-core/src/transforms/smart_crusher/crushers.rs
@@ -5,6 +5,9 @@
//! - `crush_string_array` ← `_crush_string_array` (line 2727)
//! - `crush_number_array` ← `_crush_number_array` (line 2810) — has BUG #1
//! - `crush_object` ← `_crush_object` (line 3015)
+//! Object fields are independent properties, not sampled records.
+//! `crush_object` preserves every key; nested arrays are compacted
+//! by the recursive `process_value` pass before this function runs.
//!
//! Each takes a `&SmartCrusherConfig`, a `bias` multiplier, and returns
//! `(crushed_items, strategy_string)`. Schema-preserving: the output
@@ -379,139 +382,25 @@ pub fn crush_number_array(
(kept_values, strategy)
}
-/// Crush a JSON object by selecting the most informative keys.
+/// Preserve every property of a JSON object.
///
-/// Mirrors `_crush_object`. Treats key-value pairs as items and applies
-/// `compute_optimal_k` directly on `f"{k}: {json.dumps(v)}"` strings.
-/// Always-kept rules:
-/// - keys whose value contains an error keyword.
-/// - keys with small total token estimate (<=12 tokens via the rough
-/// `len(str)/4 + len(key)/4 + 2` heuristic).
-/// - first K_first and last K_last keys (insertion order — `IndexMap`
-/// preserves it via the `serde_json/preserve_order` feature).
+/// Object fields are independent named properties, not interchangeable
+/// sampled records. The previous key-selection algorithm dropped keys
+/// (no CCR marker, no recovery hash) which silently deleted structured
+/// fields such as `tags` and `subtasks` from MCP task objects. Nested
+/// arrays are still compacted by the recursive `process_value` pass
+/// *before* this function runs, so legitimate large-array compaction
+/// and CCR-backed row offload remain available without deleting their
+/// enclosing keys.
+///
+/// `config` and `bias` are retained so the production call site in
+/// `SmartCrusher::process_value_with_hook` stays unchanged.
pub fn crush_object(
obj: &Map<String, Value>,
- config: &SmartCrusherConfig,
- bias: f64,
+ _config: &SmartCrusherConfig,
+ _bias: f64,
) -> (Map<String, Value>, String) {
- let n = obj.len();
- if n <= 8 {
- return (obj.clone(), "object:passthrough".to_string());
- }
-
- // Estimate tokens per key-value pair. Python: `len(str)/4 + len(key)/4 + 2`.
- let mut kv_tokens: Vec<(String, usize)> = Vec::with_capacity(n);
- let mut total_tokens: usize = 0;
- for (key, val) in obj {
- let val_str = serde_json::to_string(val).unwrap_or_default();
- let tokens = val_str.len() / 4 + key.len() / 4 + 2;
- kv_tokens.push((key.clone(), tokens));
- total_tokens += tokens;
- }
-
- if total_tokens < config.min_tokens_to_crush {
- return (obj.clone(), "object:passthrough".to_string());
- }
-
- // Strict lossless mode: dropped keys get no CCR marker, so keep them all.
- if config.lossless_only {
- return (obj.clone(), "object:lossless_only".to_string());
- }
-
- // Compute adaptive K on key-value string representations.
- let keys: Vec<&String> = obj.keys().collect();
- let kv_strings: Vec<String> = keys
- .iter()
- .map(|k| {
- format!(
- "{}: {}",
- k,
- serde_json::to_string(&obj[k.as_str()]).unwrap_or_default()
- )
- })
- .collect();
- let kv_refs: Vec<&str> = kv_strings.iter().map(|s| s.as_str()).collect();
-
- let max_k = if config.max_items_after_crush > 0 {
- Some(config.max_items_after_crush)
- } else {
- None
- };
- let k_total = compute_optimal_k(&kv_refs, bias, 3, max_k);
-
- if k_total >= n {
- return (obj.clone(), "object:passthrough".to_string());
- }
-
- // Always keep: error-keyword values.
- let mut keep_keys: HashSet<String> = HashSet::new();
- for (key, val) in obj {
- let val_str = serde_json::to_string(val)
- .unwrap_or_default()
- .to_lowercase();
- if ERROR_KEYWORDS.iter().any(|kw| val_str.contains(kw)) {
- keep_keys.insert(key.clone());
- }
- }
-
- // Always keep: small values (cheap to keep).
- // Python: `if tokens <= small_threshold // 4` where small_threshold=50,
- // so tokens <= 12.
- let small_threshold_tokens = 50_usize / 4;
- for (key, tokens) in &kv_tokens {
- if *tokens <= small_threshold_tokens {
- keep_keys.insert(key.clone());
- }
- }
-
- // Boundary: first K_first and last K_last (over the key insertion order).
- let k_first = 1_usize.max(round_ties_even(k_total as f64 * config.first_fraction) as usize);
- let k_last = 1_usize.max(round_ties_even(k_total as f64 * config.last_fraction) as usize);
- for k in keys.iter().take(k_first) {
- keep_keys.insert((*k).clone());
- }
- for k in keys.iter().rev().take(k_last) {
- keep_keys.insert((*k).clone());
- }
-
- // Stride fill. Python's cap recomputes the error-keyword count each
- // iteration (inefficient but deterministic). We can compute once
- // because once a key is in keep_keys, the count of error-flagged
- // entries grows monotonically — which means the cap effectively
- // grows. Mirror Python's behavior by recomputing.
- let remaining = k_total.saturating_sub(keep_keys.len());
- if remaining > 0 {
- let stride = ((n.saturating_sub(1)) / (remaining + 1)).max(1);
- let mut i: usize = 0;
- while i < n {
- // Python: `if len(keep_keys) >= k_total + len([k for k in keep_keys if any(kw in json.dumps(obj[k]).lower() for kw in keywords)])`
- let error_kept_count = keep_keys
- .iter()
- .filter(|k| {
- let s = serde_json::to_string(&obj[k.as_str()])
- .unwrap_or_default()
- .to_lowercase();
- ERROR_KEYWORDS.iter().any(|kw| s.contains(kw))
- })
- .count();
- if keep_keys.len() >= k_total + error_kept_count {
- break;
- }
- keep_keys.insert(keys[i].clone());
- i += stride;
- }
- }
-
- // Build output preserving original key insertion order.
- let mut result: Map<String, Value> = Map::new();
- for k in &keys {
- if keep_keys.contains(k.as_str()) {
- result.insert((*k).clone(), obj[k.as_str()].clone());
- }
- }
-
- let strategy = format!("object:adaptive({}->{} keys)", n, result.len());
- (result, strategy)
+ (obj.clone(), "object:passthrough".to_string())
}
// ---------- helpers ----------
@@ -747,6 +636,61 @@ mod tests {
// ---------- crush_object ----------
+ /// Wide object above the historical n>8 / token-floor gate: arbitrary
+ /// names, long strings, a nested dict, nulls, booleans, empty arrays.
+ /// `expensive_index` places the nested dict at begin / middle / end.
+ fn wide_object(expensive_index: usize) -> Map<String, Value> {
+ let n = 15;
+ let expensive_index = expensive_index.min(n - 1);
+ let mut obj = Map::new();
+ for i in 0..n {
+ if i == expensive_index {
+ obj.insert(
+ format!("expensive_{i}"),
+ json!({
+ "nested": {
+ "label": format!("nested-{i}"),
+ "note": "x".repeat(120)
+ },
+ "items": [],
+ "ok": true,
+ "missing": null
+ }),
+ );
+ } else {
+ obj.insert(
+ format!("field_{i:02}"),
+ json!(format!(
+ "this is a relatively long value string for entry number {i} with content"
+ )),
+ );
+ }
+ }
+ obj.insert("flag".to_string(), json!(false));
+ obj.insert("empty".to_string(), json!([]));
+ obj.insert("nada".to_string(), json!(null));
+ obj
+ }
+
+ fn assert_object_fully_preserved(obj: &Map<String, Value>, config: &SmartCrusherConfig) {
+ let (out, strat) = crush_object(obj, config, 1.0);
+ assert_eq!(out, *obj, "crush_object must keep every property");
+ assert_eq!(out.len(), obj.len());
+ for key in obj.keys() {
+ assert!(out.contains_key(key), "dropped key {key}");
+ assert_eq!(out.get(key), obj.get(key));
+ }
+ assert_eq!(strat, "object:passthrough");
+ assert!(
+ !strat.contains("adaptive"),
+ "must not report a key-drop strategy: {strat}"
+ );
+ assert!(
+ !strat.contains("ccr"),
+ "must not invent a CCR hash: {strat}"
+ );
+ }
+
#[test]
fn object_passthrough_when_few_keys() {
let mut obj = Map::new();
@@ -755,25 +699,29 @@ mod tests {
}
let (out, strat) = crush_object(&obj, &cfg(), 1.0);
assert_eq!(out.len(), 5);
+ assert_eq!(out, obj);
assert_eq!(strat, "object:passthrough");
}
#[test]
fn object_passthrough_when_total_tokens_below_min() {
- // Many tiny keys/values: total_tokens stays below
- // min_tokens_to_crush=200.
+ // Many tiny keys/values: historically total_tokens stayed below
+ // min_tokens_to_crush=200 and skipped selection. Preservation is
+ // now unconditional, so every key still survives.
let mut obj = Map::new();
for i in 0..30 {
obj.insert(format!("k{}", i), json!(i));
}
- let (_out, strat) = crush_object(&obj, &cfg(), 1.0);
+ let (out, strat) = crush_object(&obj, &cfg(), 1.0);
+ assert_eq!(out, obj);
assert_eq!(strat, "object:passthrough");
}
#[test]
- fn object_crushes_when_token_budget_exceeded() {
- // 30 keys, each with a long string value → total tokens > 200,
- // and unique k_total < n → actual crushing happens.
+ fn object_preserves_every_key_when_token_budget_exceeded() {
+ // 30 keys, each with a long string value → historically above
+ // the token floor and subject to key-selection. Every property
+ // must survive; no adaptive key-drop strategy.
let mut obj = Map::new();
for i in 0..30 {
obj.insert(
@@ -784,20 +732,27 @@ mod tests {
)),
);
}
- let (out, strat) = crush_object(&obj, &cfg(), 1.0);
- // Either the optimizer kept all (if it deems them all distinct
- // enough — strategy = passthrough), or it crushed.
- if strat == "object:passthrough" {
- assert_eq!(out.len(), 30);
- } else {
- assert!(strat.starts_with("object:adaptive("));
- assert!(out.len() <= 30);
+ assert_object_fully_preserved(&obj, &cfg());
+ }
+
+ #[test]
+ fn object_preserves_wide_object_with_mixed_value_types() {
+ assert_object_fully_preserved(&wide_object(7), &cfg());
+ }
+
+ #[test]
+ fn object_preserves_expensive_fields_in_any_position() {
+ // Reorder the expensive nested field into beginning, middle, and
+ // end. Boundary/stride selection used to drop the middle one.
+ for expensive_index in [0, 7, 14] {
+ assert_object_fully_preserved(&wide_object(expensive_index), &cfg());
+ assert_object_fully_preserved(&wide_object(expensive_index), &lossless_only_cfg());
}
}
#[test]
fn object_keeps_small_values() {
- // Mix of small + large values; small ones (<=12 tokens) always survive.
+ // Mix of small + large values; every key survives, including tiny.
let mut obj = Map::new();
obj.insert("tiny".to_string(), json!(1));
for i in 0..30 {
@@ -810,6 +765,7 @@ mod tests {
);
}
let (out, _) = crush_object(&obj, &cfg(), 1.0);
+ assert_eq!(out, obj);
assert!(
out.contains_key("tiny"),
"tiny key (small value) must survive"
@@ -830,6 +786,7 @@ mod tests {
);
}
let (out, _) = crush_object(&obj, &cfg(), 1.0);
+ assert_eq!(out, obj);
assert!(
out.contains_key("msg1"),
"key with error-keyword value must survive"
@@ -887,12 +844,10 @@ mod tests {
);
}
- let (lossy, _) = crush_object(&obj, &cfg(), 1.0);
- assert!(lossy.len() < obj.len(), "default config should drop keys");
-
- let (out, strat) = crush_object(&obj, &lossless_only_cfg(), 1.0);
- assert_eq!(out, obj, "lossless_only must keep every key");
- assert_eq!(strat, "object:lossless_only");
+ // Default mode used to drop keys with no CCR marker. Preservation
+ // is now unconditional, so both configs keep every property.
+ assert_object_fully_preserved(&obj, &cfg());
+ assert_object_fully_preserved(&obj, &lossless_only_cfg());
}
// ---------- BUG #1 documentation test ----------
diff --git a/tests/test_content_router_compact_json.py b/tests/test_content_router_compact_json.py
index f1b4cbe80..7cb496c6a 100644
--- a/tests/test_content_router_compact_json.py
+++ b/tests/test_content_router_compact_json.py
@@ -69,3 +69,254 @@ def test_compact_json_tool_result_compresses() -> None:
)
after = tok.count_messages(result.messages)
assert after < before * 0.9, (before, after, result.transforms_applied[:5])
+
+
+# ---------------------------------------------------------------------------
+# Issue #3634: SmartCrusher must not drop object fields (tags/subtasks)
+# ---------------------------------------------------------------------------
+
+# Exact MCP `saga-mcp_task_get` payload from the issue (813 bytes, original
+# key order, tags encoded as a JSON string). Do not pretty-print or reorder.
+MCP_TASK_GET_JSON = (
+ '{"id":79,"epic_id":9,"title":"test","description":null,'
+ '"status":"done","priority":"medium","sort_order":0,'
+ '"assigned_to":null,"estimated_hours":null,"actual_hours":null,'
+ '"due_date":null,"source_ref":null,"metadata":"{}",'
+ '"created_at":"2026-09-15 20:32:06","updated_at":"2026-09-16 11:59:27",'
+ '"description_locked":0,"is_deleted":0,"deleted_at":null,'
+ '"deleted_by":null,"delete_reason":null,"epic_name":"latency-routing",'
+ '"tags":"[\\"cherry-pick\\",\\"dedicated branch\\"]",'
+ '"subtasks":[{"id":163,"task_id":79,"title":"test2","status":"todo",'
+ '"sort_order":1,"created_at":"2026-09-15 20:32:13",'
+ '"updated_at":"2026-09-16 14:02:26"},'
+ '{"id":165,"task_id":79,"title":"test4","status":"todo","sort_order":2,'
+ '"created_at":"2026-09-15 20:38:47","updated_at":"2026-09-17 20:29:04"}],'
+ '"notes":[],"comments":[],"depends_on":[],"dependents":[]}'
+)
+
+_TOOL_CALL_ID = "call_task_get_79"
+_TOOL_NAME = "saga-mcp_task_get"
+
+
+def _offline_tokenizer():
+ tokenizer = pytest.importorskip("headroom.tokenizers.estimator")
+ from headroom.tokenizer import Tokenizer
+
+ return Tokenizer(tokenizer.EstimatingTokenCounter())
+
+
+def _object_preserving_router() -> ContentRouter:
+ # Real ContentRouter + Rust SmartCrusher. Disable unrelated ML / log
+ # fallbacks so a no-savings crush cannot be rewritten by Kompress.
+ return ContentRouter(
+ ContentRouterConfig(
+ skip_user_messages=False,
+ enable_kompress=False,
+ enable_log_compressor=False,
+ enable_search_compressor=False,
+ relevance_split=False,
+ protect_analysis_context=False,
+ exclude_tools=set(),
+ )
+ )
+
+
+def _openai_task_messages(payload: str) -> list[dict]:
+ return [
+ {"role": "user", "content": "read task 79"},
+ {
+ "role": "assistant",
+ "content": None,
+ "tool_calls": [
+ {
+ "id": _TOOL_CALL_ID,
+ "type": "function",
+ "function": {"name": _TOOL_NAME, "arguments": '{"id":79}'},
+ }
+ ],
+ },
+ {
+ "role": "tool",
+ "tool_call_id": _TOOL_CALL_ID,
+ "content": payload,
+ },
+ ]
+
+
+def _anthropic_task_messages(payload: str) -> list[dict]:
+ return [
+ {"role": "user", "content": "read task 79"},
+ {
+ "role": "assistant",
+ "content": [
+ {
+ "type": "tool_use",
+ "id": _TOOL_CALL_ID,
+ "name": _TOOL_NAME,
+ "input": {"id": 79},
+ }
+ ],
+ },
+ {
+ "role": "user",
+ "content": [
+ {
+ "type": "tool_result",
+ "tool_use_id": _TOOL_CALL_ID,
+ "content": payload,
+ }
+ ],
+ },
+ ]
+
+
+def _apply(messages: list[dict]):
+ router = _object_preserving_router()
+ tok = _offline_tokenizer()
+ return router.apply(
+ [json.loads(json.dumps(m)) for m in messages],
+ tok,
+ context="read task 79",
+ frozen_message_count=0,
+ protect_recent=0,
+ protect_analysis_context=False,
+ )
+
+
+def _openai_tool_content(messages: list[dict]) -> str:
+ for msg in messages:
+ if msg.get("role") == "tool" and msg.get("tool_call_id") == _TOOL_CALL_ID:
+ content = msg.get("content")
+ assert isinstance(content, str), content
+ return content
+ raise AssertionError(f"OpenAI tool message {_TOOL_CALL_ID} missing")
+
+
+def _anthropic_tool_content(messages: list[dict]) -> str:
+ for msg in messages:
+ content = msg.get("content")
+ if not isinstance(content, list):
+ continue
+ for block in content:
+ if block.get("type") == "tool_result" and block.get("tool_use_id") == _TOOL_CALL_ID:
+ inner = block.get("content")
+ assert isinstance(inner, str), inner
+ return inner
+ raise AssertionError(f"Anthropic tool_result {_TOOL_CALL_ID} missing")
+
+
+def _assert_task_payload_preserved(original: str, compressed: str) -> dict:
+ assert "<<ccr:" not in compressed, compressed[:200]
+ parsed = json.loads(compressed)
+ expected = json.loads(original)
+ assert parsed == expected, (sorted(expected), sorted(parsed))
+ assert "tags" in parsed
+ assert "subtasks" in parsed
+ assert parsed["tags"] == '["cherry-pick","dedicated branch"]'
+ assert len(parsed["subtasks"]) == 2
+ assert parsed["subtasks"][0]["title"] == "test2"
+ assert parsed["subtasks"][1]["title"] == "test4"
+ return parsed
+
+
+def _wide_renamed_task_json() -> str:
+ """Renamed, oversized variant so a below-threshold skip cannot hide key drop.
+
+ Field names are arbitrary (not tags/subtasks) and expensive values sit at
+ the beginning, middle, and end so a boundary/stride selector cannot
+ accidentally keep them all.
+ """
+ original = json.loads(MCP_TASK_GET_JSON)
+ expensive = {
+ "nested": {"label": "child", "note": "n" * 180},
+ "items": [],
+ "ok": True,
+ "missing": None,
+ }
+ wide: dict = {
+ "leading_blob": expensive,
+ "label_bundle": original.pop("tags"),
+ "child_records": original.pop("subtasks"),
+ }
+ wide.update(original)
+ for i in range(20):
+ wide[f"extra_field_{i:02d}"] = (
+ f"this is a relatively long value string for entry number {i} with content"
+ )
+ wide["trailing_blob"] = {
+ "nested": {"label": "tail", "note": "t" * 180},
+ "items": [],
+ "ok": False,
+ "missing": None,
+ }
+ payload = json.dumps(wide, separators=(",", ":"))
+ assert len(payload) > 1500
+ assert len(wide) > 8
+ return payload
+
+
+def test_mcp_task_fixture_is_the_issue_payload() -> None:
+ assert len(MCP_TASK_GET_JSON) == 813
+ parsed = json.loads(MCP_TASK_GET_JSON)
+ assert list(parsed)[21] == "tags"
+ assert list(parsed)[22] == "subtasks"
+ assert parsed["tags"] == '["cherry-pick","dedicated branch"]'
+
+
+def test_mcp_task_object_preserved_openai_tool_role() -> None:
+ messages = _openai_task_messages(MCP_TASK_GET_JSON)
+ result = _apply(messages)
+ content = _openai_tool_content(result.messages)
+ assert content == MCP_TASK_GET_JSON
+ _assert_task_payload_preserved(MCP_TASK_GET_JSON, content)
+ # Tool-call pairing stays intact.
+ tool_msg = next(m for m in result.messages if m.get("role") == "tool")
+ assistant = next(m for m in result.messages if m.get("role") == "assistant")
+ assert tool_msg["tool_call_id"] == _TOOL_CALL_ID
+ assert assistant["tool_calls"][0]["id"] == _TOOL_CALL_ID
+ assert assistant["tool_calls"][0]["function"]["name"] == _TOOL_NAME
+ assert not any("object:adaptive" in t for t in result.transforms_applied)
+
+
+def test_mcp_task_object_preserved_anthropic_tool_result() -> None:
+ messages = _anthropic_task_messages(MCP_TASK_GET_JSON)
+ result = _apply(messages)
+ content = _anthropic_tool_content(result.messages)
+ assert content == MCP_TASK_GET_JSON
+ _assert_task_payload_preserved(MCP_TASK_GET_JSON, content)
+ assistant = next(m for m in result.messages if m.get("role") == "assistant")
+ assert assistant["content"][0]["id"] == _TOOL_CALL_ID
+ user_blocks = next(
+ m["content"]
+ for m in result.messages
+ if m.get("role") == "user" and isinstance(m.get("content"), list)
+ )
+ assert user_blocks[0]["tool_use_id"] == _TOOL_CALL_ID
+ assert not any("object:adaptive" in t for t in result.transforms_applied)
+
+
+def test_wide_renamed_object_fields_preserved_openai_tool_role() -> None:
+ payload = _wide_renamed_task_json()
+ result = _apply(_openai_task_messages(payload))
+ content = _openai_tool_content(result.messages)
+ assert "<<ccr:" not in content
+ parsed = json.loads(content)
+ expected = json.loads(payload)
+ assert parsed == expected
+ assert "label_bundle" in parsed
+ assert "child_records" in parsed
+ assert "leading_blob" in parsed
+ assert "trailing_blob" in parsed
+ assert parsed["label_bundle"] == '["cherry-pick","dedicated branch"]'
+ assert len(parsed["child_records"]) == 2
+ assert parsed["leading_blob"]["nested"]["note"] == "n" * 180
+ assert parsed["trailing_blob"]["nested"]["note"] == "t" * 180
+
+
+def test_malformed_json_tool_result_passes_through() -> None:
+ payload = "{" + "not-json, still a tool result. " * 40
+ assert len(payload) > 500
+ result = _apply(_openai_task_messages(payload))
+ content = _openai_tool_content(result.messages)
+ assert content == payload
+ assert "<<ccr:" not in content