Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix: preserve JSON object fields in SmartCrusher (#3648)
Matt Van Horn545f54414703
diff --git a/crates/headroom-core/src/transforms/smart_crusher/crusher.rs b/crates/headroom-core/src/transforms/smart_crusher/crusher.rsindex a247890a2..9d71197ec 100644--- a/crates/headroom-core/src/transforms/smart_crusher/crusher.rs+++ b/crates/headroom-core/src/transforms/smart_crusher/crusher.rs@@ -446,10 +446,26 @@ impl SmartCrusher {let (crushed, info) =self.process_value_with_hook(&parsed, 0, query_context, bias, prose_hook);+ // When an object and its descendants were left unchanged, keep+ // the original bytes. Re-serializing would rewrite Unicode+ // escapes and numeric lexical forms, then advertise a false+ // compression. Nested skip/compaction still serializes below+ // because those paths populate `info`. Spaced JSON without a+ // nested strategy is compact-serialized for historical crush()+ // / parity fixtures.+ if crushed == parsed+ && info.is_empty()+ && matches!(parsed, Value::Object(_))+ && !has_json_insignificant_whitespace(content)+ {+ return (content.to_string(), false, String::new());+ }+// Re-serialize with Python `safe_json_dumps` formatting:// compact `(",", ":")` separators + `ensure_ascii=False`,// preserving object-key insertion order. Matches the Python- // SmartCrusher output bytes the proxy writes.+ // SmartCrusher output bytes the proxy writes. When descendants+ // actually change, this path keeps strategy attribution.let result = crate::transforms::anchor_selector::python_safe_json_dumps(&crushed);let was_modified = result != content.trim();(result, was_modified, info)@@ -629,8 +645,11 @@ impl SmartCrusher {}}- // Second pass: if the object itself has many keys,- // compress at the key level.+ // Second pass: crush_object preserves every property+ // (object fields are not interchangeable sampled+ // records). Nested arrays were already compacted above,+ // so large-array compaction and CCR row offload still+ // run without deleting their enclosing keys.if processed.len() >= self.config.min_items_to_analyze {let (crushed_dict, strategy) = crush_object(&processed, &self.config, bias);if strategy != "object:passthrough" {@@ -1213,6 +1232,36 @@ fn hash_canonical(canonical: &str) -> String {// here because `process_string`'s `string_ccr:<kind>` strategy-info// label is local to this module's debug-string convention.+/// True when `s` has JSON whitespace outside of string literals+/// (spaces after `:` / `,`, pretty-print newlines, etc.). Used to+/// keep compact unchanged objects on the original-bytes path while+/// still compact-serializing spaced inputs for historical crush()+/// output.+fn has_json_insignificant_whitespace(s: &str) -> bool {+ let mut in_string = false;+ let mut escape = false;+ for ch in s.trim().chars() {+ if in_string {+ if escape {+ escape = false;+ } else if ch == '\\' {+ escape = true;+ } else if ch == '"' {+ in_string = false;+ }+ continue;+ }+ if ch == '"' {+ in_string = true;+ continue;+ }+ if ch.is_whitespace() {+ return true;+ }+ }+ false+}+fn opaque_kind_label(kind: &super::compaction::OpaqueKind) -> &str {use super::compaction::OpaqueKind;match kind {@@ -1533,18 +1582,70 @@ mod tests {}#[test]- fn crush_serializes_with_python_safe_format() {+ fn crush_serializes_changed_objects_with_python_safe_format() {let c = crusher();- // SmartCrusher uses Python's `safe_json_dumps`: compact- // separators `(",", ":")` + `ensure_ascii=False`, preserving- // object-key insertion order. A spaced input round-trips to- // the compact form.- let input = r#"{"a": 1, "b": 2, "c": 3}"#;- let result = c.crush(input, "", 1.0);- assert_eq!(- result.compressed, r#"{"a":1,"b":2,"c":3}"#,- "safe_json_dumps emits compact `,` / `:` separators"+ // When descendants change, SmartCrusher re-serializes with+ // Python's `safe_json_dumps`: compact separators `(",", ":")`+ // + `ensure_ascii=False`. Unchanged objects keep original+ // bytes instead of being rewritten.+ let mut inner = String::from("[");+ for i in 0..30 {+ if i > 0 {+ inner.push(',');+ }+ inner.push_str(r#"{"status": "ok"}"#);+ }+ inner.push(']');+ let input = format!(r#"{{"a": 1, "data": {inner}}}"#);+ let result = c.crush(&input, "", 1.0);+ assert!(+ result.was_modified,+ "nested compressible array must trigger serialization"+ );+ assert!(+ result.compressed.contains(r#""a":1"#),+ "safe_json_dumps emits compact `:`; got {}",+ result.compressed+ );+ assert!(+ !result.compressed.contains(": "),+ "changed output must not keep spaced separators: {}",+ result.compressed+ );+ assert_ne!(result.strategy, "passthrough");+ }++ #[test]+ fn crush_unchanged_object_keeps_original_bytes() {+ let c = crusher();+ // Compact input: escaped Unicode and numeric lexical forms must+ // survive the passthrough branch rather than being reserialized+ // (`\u00e9` → `é`, `1.0` → `1`).+ let compact = r#"{"cafe":"caf\u00e9","n":1.0,"ok":true}"#;+ let result = c.crush(compact, "", 1.0);+ assert_eq!(result.compressed, compact);+ assert!(!result.was_modified);+ assert_eq!(result.strategy, "passthrough");+ assert!(+ !result.compressed.contains("<<ccr:"),+ "unchanged object must not invent a CCR marker");++ // Spaced / pretty-printed unchanged objects still compact-serialize+ // (historical crush() output, required by parity fixtures) but+ // must not advertise a key-drop strategy or invent a CCR hash.+ let spaced = "{ \"cafe\": \"caf\\u00e9\", \"n\": 1.0, \"ok\": true }";+ let spaced_result = c.crush(spaced, "", 1.0);+ assert_eq!(spaced_result.strategy, "passthrough");+ assert!(+ !spaced_result.strategy.contains("adaptive"),+ "must not report a key-drop strategy: {}",+ spaced_result.strategy+ );+ assert!(!spaced_result.compressed.contains("<<ccr:"));+ let parsed: Value = serde_json::from_str(&spaced_result.compressed).unwrap();+ assert_eq!(parsed["cafe"], "café");+ assert_eq!(parsed["ok"], true);}#[test]@@ -1552,6 +1653,7 @@ mod tests {let c = crusher();// Top-level dict with a nested array of 30 identical items.// The inner array should compress (low_uniqueness path).+ // Sibling fields stay even though the array is rewritten.let mut inner = String::from("[");for i in 0..30 {if i > 0 {@@ -1560,12 +1662,105 @@ mod tests {inner.push_str(r#"{"status":"ok"}"#);}inner.push(']');- let input = format!(r#"{{"data": {}}}"#, inner);+ let input =+ format!(r#"{{"label":"keep","data":{inner},"empty":[],"flag":false,"missing":null}}"#);let result = c.crush(&input, "", 1.0);assert!(result.was_modified,"nested compressible array must be crushed even inside a wrapper object");+ let parsed: Value = serde_json::from_str(&result.compressed).expect("output is JSON");+ assert_eq!(parsed["label"], "keep");+ assert_eq!(parsed["empty"], json!([]));+ assert_eq!(parsed["flag"], false);+ assert_eq!(parsed["missing"], Value::Null);+ assert!(+ parsed.get("data").is_some(),+ "enclosing array property must survive"+ );+ }++ /// Issue #3634 MCP `saga-mcp_task_get` payload (813 bytes, original+ /// key order, tags encoded as a JSON string).+ const MCP_TASK_GET_JSON: &str = r#"{"id":79,"epic_id":9,"title":"test","description":null,"status":"done","priority":"medium","sort_order":0,"assigned_to":null,"estimated_hours":null,"actual_hours":null,"due_date":null,"source_ref":null,"metadata":"{}","created_at":"2026-09-15 20:32:06","updated_at":"2026-09-16 11:59:27","description_locked":0,"is_deleted":0,"deleted_at":null,"deleted_by":null,"delete_reason":null,"epic_name":"latency-routing","tags":"[\"cherry-pick\",\"dedicated branch\"]","subtasks":[{"id":163,"task_id":79,"title":"test2","status":"todo","sort_order":1,"created_at":"2026-09-15 20:32:13","updated_at":"2026-09-16 14:02:26"},{"id":165,"task_id":79,"title":"test4","status":"todo","sort_order":2,"created_at":"2026-09-15 20:38:47","updated_at":"2026-09-17 20:29:04"}],"notes":[],"comments":[],"depends_on":[],"dependents":[]}"#;++ #[test]+ fn crush_preserves_mcp_task_object_fields() {+ let c = crusher();+ let result = c.crush(MCP_TASK_GET_JSON, "", 1.0);+ assert_eq!(+ result.compressed, MCP_TASK_GET_JSON,+ "compact unchanged object must stay byte-identical"+ );+ assert!(!result.was_modified);+ assert_eq!(result.strategy, "passthrough");+ assert!(+ !result.strategy.contains("adaptive"),+ "must not report a key-drop strategy: {}",+ result.strategy+ );+ assert!(+ !result.compressed.contains("<<ccr:"),+ "must not invent a CCR marker"+ );+ let parsed: Value = serde_json::from_str(&result.compressed).expect("output is JSON");+ assert_eq!(parsed["tags"], "[\"cherry-pick\",\"dedicated branch\"]");+ let subtasks = parsed["subtasks"].as_array().expect("subtasks array");+ assert_eq!(subtasks.len(), 2);+ assert_eq!(subtasks[0]["title"], "test2");+ assert_eq!(subtasks[1]["title"], "test4");+ assert_eq!(parsed["description"], Value::Null);+ assert_eq!(parsed["notes"], json!([]));+ }++ #[test]+ fn nested_array_offload_keeps_enclosing_object_keys() {+ // Force the lossy row-drop path so CCR offload fires. The+ // enclosing object must keep every sibling field, and the+ // stored payload must round-trip as the original rows.+ let cfg = SmartCrusherConfig {+ lossless_min_savings_ratio: 0.99,+ ..SmartCrusherConfig::default()+ };+ let c = SmartCrusher::new(cfg);+ let rows: Vec<Value> = (0..50).map(|_| json!({"status": "ok"})).collect();+ let doc = json!({+ "keep_me": "sibling",+ "rows": rows,+ "also_keep": null,+ "flag": true,+ "empty": []+ });+ let content = serde_json::to_string(&doc).unwrap();+ let result = c.crush(&content, "", 1.0);+ assert!(result.was_modified, "lossy nested array should compress");+ assert_ne!(result.strategy, "passthrough");+ let parsed: Value = serde_json::from_str(&result.compressed).expect("output is JSON");+ assert_eq!(parsed["keep_me"], "sibling");+ assert_eq!(parsed["also_keep"], Value::Null);+ assert_eq!(parsed["flag"], true);+ assert_eq!(parsed["empty"], json!([]));+ assert!(+ parsed.get("rows").is_some(),+ "enclosing array property must survive"+ );+ assert!(+ result.compressed.contains("<<ccr:"),+ "offload must emit a CCR marker: {}",+ result.compressed+ );+ let marker_at = result+ .compressed+ .find("<<ccr:")+ .expect("CCR marker present");+ let hash = &result.compressed[marker_at + 6..marker_at + 18];+ let stored = c+ .ccr_store()+ .expect("default crusher has a store")+ .get(hash)+ .expect("offloaded rows are retrievable");+ let recovered: Value = serde_json::from_str(&stored).expect("stored JSON");+ assert_eq!(recovered, Value::Array(rows));}#[test]@@ -2134,6 +2329,12 @@ mod tests {let lossy: Value = serde_json::from_str(&crusher().crush(&content, "", 1.0).compressed).expect("default output is JSON");assert_ne!(lossy, doc, "default config should drop items");+ // Object keys are not sampled records: default mode may crush+ // nested arrays but must keep every property of `meta`.+ assert_eq!(+ lossy["meta"], doc["meta"],+ "default mode must keep object keys"+ );let strict = SmartCrusher::new(SmartCrusherConfig {lossless_only: true,diff --git a/crates/headroom-core/src/transforms/smart_crusher/crushers.rs b/crates/headroom-core/src/transforms/smart_crusher/crushers.rsindex a6f3ecec2..1d203a8fd 100644--- a/crates/headroom-core/src/transforms/smart_crusher/crushers.rs+++ b/crates/headroom-core/src/transforms/smart_crusher/crushers.rs@@ -5,6 +5,9 @@//! - `crush_string_array` ← `_crush_string_array` (line 2727)//! - `crush_number_array` ← `_crush_number_array` (line 2810) — has BUG #1//! - `crush_object` ← `_crush_object` (line 3015)+//! Object fields are independent properties, not sampled records.+//! `crush_object` preserves every key; nested arrays are compacted+//! by the recursive `process_value` pass before this function runs.//!//! Each takes a `&SmartCrusherConfig`, a `bias` multiplier, and returns//! `(crushed_items, strategy_string)`. Schema-preserving: the output@@ -379,139 +382,25 @@ pub fn crush_number_array((kept_values, strategy)}-/// Crush a JSON object by selecting the most informative keys.+/// Preserve every property of a JSON object.///-/// Mirrors `_crush_object`. Treats key-value pairs as items and applies-/// `compute_optimal_k` directly on `f"{k}: {json.dumps(v)}"` strings.-/// Always-kept rules:-/// - keys whose value contains an error keyword.-/// - keys with small total token estimate (<=12 tokens via the rough-/// `len(str)/4 + len(key)/4 + 2` heuristic).-/// - first K_first and last K_last keys (insertion order — `IndexMap`-/// preserves it via the `serde_json/preserve_order` feature).+/// Object fields are independent named properties, not interchangeable+/// sampled records. The previous key-selection algorithm dropped keys+/// (no CCR marker, no recovery hash) which silently deleted structured+/// fields such as `tags` and `subtasks` from MCP task objects. Nested+/// arrays are still compacted by the recursive `process_value` pass+/// *before* this function runs, so legitimate large-array compaction+/// and CCR-backed row offload remain available without deleting their+/// enclosing keys.+///+/// `config` and `bias` are retained so the production call site in+/// `SmartCrusher::process_value_with_hook` stays unchanged.pub fn crush_object(obj: &Map<String, Value>,- config: &SmartCrusherConfig,- bias: f64,+ _config: &SmartCrusherConfig,+ _bias: f64,) -> (Map<String, Value>, String) {- let n = obj.len();- if n <= 8 {- return (obj.clone(), "object:passthrough".to_string());- }-- // Estimate tokens per key-value pair. Python: `len(str)/4 + len(key)/4 + 2`.- let mut kv_tokens: Vec<(String, usize)> = Vec::with_capacity(n);- let mut total_tokens: usize = 0;- for (key, val) in obj {- let val_str = serde_json::to_string(val).unwrap_or_default();- let tokens = val_str.len() / 4 + key.len() / 4 + 2;- kv_tokens.push((key.clone(), tokens));- total_tokens += tokens;- }-- if total_tokens < config.min_tokens_to_crush {- return (obj.clone(), "object:passthrough".to_string());- }-- // Strict lossless mode: dropped keys get no CCR marker, so keep them all.- if config.lossless_only {- return (obj.clone(), "object:lossless_only".to_string());- }-- // Compute adaptive K on key-value string representations.- let keys: Vec<&String> = obj.keys().collect();- let kv_strings: Vec<String> = keys- .iter()- .map(|k| {- format!(- "{}: {}",- k,- serde_json::to_string(&obj[k.as_str()]).unwrap_or_default()- )- })- .collect();- let kv_refs: Vec<&str> = kv_strings.iter().map(|s| s.as_str()).collect();-- let max_k = if config.max_items_after_crush > 0 {- Some(config.max_items_after_crush)- } else {- None- };- let k_total = compute_optimal_k(&kv_refs, bias, 3, max_k);-- if k_total >= n {- return (obj.clone(), "object:passthrough".to_string());- }-- // Always keep: error-keyword values.- let mut keep_keys: HashSet<String> = HashSet::new();- for (key, val) in obj {- let val_str = serde_json::to_string(val)- .unwrap_or_default()- .to_lowercase();- if ERROR_KEYWORDS.iter().any(|kw| val_str.contains(kw)) {- keep_keys.insert(key.clone());- }- }-- // Always keep: small values (cheap to keep).- // Python: `if tokens <= small_threshold // 4` where small_threshold=50,- // so tokens <= 12.- let small_threshold_tokens = 50_usize / 4;- for (key, tokens) in &kv_tokens {- if *tokens <= small_threshold_tokens {- keep_keys.insert(key.clone());- }- }-- // Boundary: first K_first and last K_last (over the key insertion order).- let k_first = 1_usize.max(round_ties_even(k_total as f64 * config.first_fraction) as usize);- let k_last = 1_usize.max(round_ties_even(k_total as f64 * config.last_fraction) as usize);- for k in keys.iter().take(k_first) {- keep_keys.insert((*k).clone());- }- for k in keys.iter().rev().take(k_last) {- keep_keys.insert((*k).clone());- }-- // Stride fill. Python's cap recomputes the error-keyword count each- // iteration (inefficient but deterministic). We can compute once- // because once a key is in keep_keys, the count of error-flagged- // entries grows monotonically — which means the cap effectively- // grows. Mirror Python's behavior by recomputing.- let remaining = k_total.saturating_sub(keep_keys.len());- if remaining > 0 {- let stride = ((n.saturating_sub(1)) / (remaining + 1)).max(1);- let mut i: usize = 0;- while i < n {- // Python: `if len(keep_keys) >= k_total + len([k for k in keep_keys if any(kw in json.dumps(obj[k]).lower() for kw in keywords)])`- let error_kept_count = keep_keys- .iter()- .filter(|k| {- let s = serde_json::to_string(&obj[k.as_str()])- .unwrap_or_default()- .to_lowercase();- ERROR_KEYWORDS.iter().any(|kw| s.contains(kw))- })- .count();- if keep_keys.len() >= k_total + error_kept_count {- break;- }- keep_keys.insert(keys[i].clone());- i += stride;- }- }-- // Build output preserving original key insertion order.- let mut result: Map<String, Value> = Map::new();- for k in &keys {- if keep_keys.contains(k.as_str()) {- result.insert((*k).clone(), obj[k.as_str()].clone());- }- }-- let strategy = format!("object:adaptive({}->{} keys)", n, result.len());- (result, strategy)+ (obj.clone(), "object:passthrough".to_string())}// ---------- helpers ----------@@ -747,6 +636,61 @@ mod tests {// ---------- crush_object ----------+ /// Wide object above the historical n>8 / token-floor gate: arbitrary+ /// names, long strings, a nested dict, nulls, booleans, empty arrays.+ /// `expensive_index` places the nested dict at begin / middle / end.+ fn wide_object(expensive_index: usize) -> Map<String, Value> {+ let n = 15;+ let expensive_index = expensive_index.min(n - 1);+ let mut obj = Map::new();+ for i in 0..n {+ if i == expensive_index {+ obj.insert(+ format!("expensive_{i}"),+ json!({+ "nested": {+ "label": format!("nested-{i}"),+ "note": "x".repeat(120)+ },+ "items": [],+ "ok": true,+ "missing": null+ }),+ );+ } else {+ obj.insert(+ format!("field_{i:02}"),+ json!(format!(+ "this is a relatively long value string for entry number {i} with content"+ )),+ );+ }+ }+ obj.insert("flag".to_string(), json!(false));+ obj.insert("empty".to_string(), json!([]));+ obj.insert("nada".to_string(), json!(null));+ obj+ }++ fn assert_object_fully_preserved(obj: &Map<String, Value>, config: &SmartCrusherConfig) {+ let (out, strat) = crush_object(obj, config, 1.0);+ assert_eq!(out, *obj, "crush_object must keep every property");+ assert_eq!(out.len(), obj.len());+ for key in obj.keys() {+ assert!(out.contains_key(key), "dropped key {key}");+ assert_eq!(out.get(key), obj.get(key));+ }+ assert_eq!(strat, "object:passthrough");+ assert!(+ !strat.contains("adaptive"),+ "must not report a key-drop strategy: {strat}"+ );+ assert!(+ !strat.contains("ccr"),+ "must not invent a CCR hash: {strat}"+ );+ }+#[test]fn object_passthrough_when_few_keys() {let mut obj = Map::new();@@ -755,25 +699,29 @@ mod tests {}let (out, strat) = crush_object(&obj, &cfg(), 1.0);assert_eq!(out.len(), 5);+ assert_eq!(out, obj);assert_eq!(strat, "object:passthrough");}#[test]fn object_passthrough_when_total_tokens_below_min() {- // Many tiny keys/values: total_tokens stays below- // min_tokens_to_crush=200.+ // Many tiny keys/values: historically total_tokens stayed below+ // min_tokens_to_crush=200 and skipped selection. Preservation is+ // now unconditional, so every key still survives.let mut obj = Map::new();for i in 0..30 {obj.insert(format!("k{}", i), json!(i));}- let (_out, strat) = crush_object(&obj, &cfg(), 1.0);+ let (out, strat) = crush_object(&obj, &cfg(), 1.0);+ assert_eq!(out, obj);assert_eq!(strat, "object:passthrough");}#[test]- fn object_crushes_when_token_budget_exceeded() {- // 30 keys, each with a long string value → total tokens > 200,- // and unique k_total < n → actual crushing happens.+ fn object_preserves_every_key_when_token_budget_exceeded() {+ // 30 keys, each with a long string value → historically above+ // the token floor and subject to key-selection. Every property+ // must survive; no adaptive key-drop strategy.let mut obj = Map::new();for i in 0..30 {obj.insert(@@ -784,20 +732,27 @@ mod tests {)),);}- let (out, strat) = crush_object(&obj, &cfg(), 1.0);- // Either the optimizer kept all (if it deems them all distinct- // enough — strategy = passthrough), or it crushed.- if strat == "object:passthrough" {- assert_eq!(out.len(), 30);- } else {- assert!(strat.starts_with("object:adaptive("));- assert!(out.len() <= 30);+ assert_object_fully_preserved(&obj, &cfg());+ }++ #[test]+ fn object_preserves_wide_object_with_mixed_value_types() {+ assert_object_fully_preserved(&wide_object(7), &cfg());+ }++ #[test]+ fn object_preserves_expensive_fields_in_any_position() {+ // Reorder the expensive nested field into beginning, middle, and+ // end. Boundary/stride selection used to drop the middle one.+ for expensive_index in [0, 7, 14] {+ assert_object_fully_preserved(&wide_object(expensive_index), &cfg());+ assert_object_fully_preserved(&wide_object(expensive_index), &lossless_only_cfg());}}#[test]fn object_keeps_small_values() {- // Mix of small + large values; small ones (<=12 tokens) always survive.+ // Mix of small + large values; every key survives, including tiny.let mut obj = Map::new();obj.insert("tiny".to_string(), json!(1));for i in 0..30 {@@ -810,6 +765,7 @@ mod tests {);}let (out, _) = crush_object(&obj, &cfg(), 1.0);+ assert_eq!(out, obj);assert!(out.contains_key("tiny"),"tiny key (small value) must survive"@@ -830,6 +786,7 @@ mod tests {);}let (out, _) = crush_object(&obj, &cfg(), 1.0);+ assert_eq!(out, obj);assert!(out.contains_key("msg1"),"key with error-keyword value must survive"@@ -887,12 +844,10 @@ mod tests {);}- let (lossy, _) = crush_object(&obj, &cfg(), 1.0);- assert!(lossy.len() < obj.len(), "default config should drop keys");-- let (out, strat) = crush_object(&obj, &lossless_only_cfg(), 1.0);- assert_eq!(out, obj, "lossless_only must keep every key");- assert_eq!(strat, "object:lossless_only");+ // Default mode used to drop keys with no CCR marker. Preservation+ // is now unconditional, so both configs keep every property.+ assert_object_fully_preserved(&obj, &cfg());+ assert_object_fully_preserved(&obj, &lossless_only_cfg());}// ---------- BUG #1 documentation test ----------diff --git a/tests/test_content_router_compact_json.py b/tests/test_content_router_compact_json.pyindex f1b4cbe80..7cb496c6a 100644--- a/tests/test_content_router_compact_json.py+++ b/tests/test_content_router_compact_json.py@@ -69,3 +69,254 @@ def test_compact_json_tool_result_compresses() -> None:)after = tok.count_messages(result.messages)assert after < before * 0.9, (before, after, result.transforms_applied[:5])+++# ---------------------------------------------------------------------------+# Issue #3634: SmartCrusher must not drop object fields (tags/subtasks)+# ---------------------------------------------------------------------------++# Exact MCP `saga-mcp_task_get` payload from the issue (813 bytes, original+# key order, tags encoded as a JSON string). Do not pretty-print or reorder.+MCP_TASK_GET_JSON = (+ '{"id":79,"epic_id":9,"title":"test","description":null,'+ '"status":"done","priority":"medium","sort_order":0,'+ '"assigned_to":null,"estimated_hours":null,"actual_hours":null,'+ '"due_date":null,"source_ref":null,"metadata":"{}",'+ '"created_at":"2026-09-15 20:32:06","updated_at":"2026-09-16 11:59:27",'+ '"description_locked":0,"is_deleted":0,"deleted_at":null,'+ '"deleted_by":null,"delete_reason":null,"epic_name":"latency-routing",'+ '"tags":"[\\"cherry-pick\\",\\"dedicated branch\\"]",'+ '"subtasks":[{"id":163,"task_id":79,"title":"test2","status":"todo",'+ '"sort_order":1,"created_at":"2026-09-15 20:32:13",'+ '"updated_at":"2026-09-16 14:02:26"},'+ '{"id":165,"task_id":79,"title":"test4","status":"todo","sort_order":2,'+ '"created_at":"2026-09-15 20:38:47","updated_at":"2026-09-17 20:29:04"}],'+ '"notes":[],"comments":[],"depends_on":[],"dependents":[]}'+)++_TOOL_CALL_ID = "call_task_get_79"+_TOOL_NAME = "saga-mcp_task_get"+++def _offline_tokenizer():+ tokenizer = pytest.importorskip("headroom.tokenizers.estimator")+ from headroom.tokenizer import Tokenizer++ return Tokenizer(tokenizer.EstimatingTokenCounter())+++def _object_preserving_router() -> ContentRouter:+ # Real ContentRouter + Rust SmartCrusher. Disable unrelated ML / log+ # fallbacks so a no-savings crush cannot be rewritten by Kompress.+ return ContentRouter(+ ContentRouterConfig(+ skip_user_messages=False,+ enable_kompress=False,+ enable_log_compressor=False,+ enable_search_compressor=False,+ relevance_split=False,+ protect_analysis_context=False,+ exclude_tools=set(),+ )+ )+++def _openai_task_messages(payload: str) -> list[dict]:+ return [+ {"role": "user", "content": "read task 79"},+ {+ "role": "assistant",+ "content": None,+ "tool_calls": [+ {+ "id": _TOOL_CALL_ID,+ "type": "function",+ "function": {"name": _TOOL_NAME, "arguments": '{"id":79}'},+ }+ ],+ },+ {+ "role": "tool",+ "tool_call_id": _TOOL_CALL_ID,+ "content": payload,+ },+ ]+++def _anthropic_task_messages(payload: str) -> list[dict]:+ return [+ {"role": "user", "content": "read task 79"},+ {+ "role": "assistant",+ "content": [+ {+ "type": "tool_use",+ "id": _TOOL_CALL_ID,+ "name": _TOOL_NAME,+ "input": {"id": 79},+ }+ ],+ },+ {+ "role": "user",+ "content": [+ {+ "type": "tool_result",+ "tool_use_id": _TOOL_CALL_ID,+ "content": payload,+ }+ ],+ },+ ]+++def _apply(messages: list[dict]):+ router = _object_preserving_router()+ tok = _offline_tokenizer()+ return router.apply(+ [json.loads(json.dumps(m)) for m in messages],+ tok,+ context="read task 79",+ frozen_message_count=0,+ protect_recent=0,+ protect_analysis_context=False,+ )+++def _openai_tool_content(messages: list[dict]) -> str:+ for msg in messages:+ if msg.get("role") == "tool" and msg.get("tool_call_id") == _TOOL_CALL_ID:+ content = msg.get("content")+ assert isinstance(content, str), content+ return content+ raise AssertionError(f"OpenAI tool message {_TOOL_CALL_ID} missing")+++def _anthropic_tool_content(messages: list[dict]) -> str:+ for msg in messages:+ content = msg.get("content")+ if not isinstance(content, list):+ continue+ for block in content:+ if block.get("type") == "tool_result" and block.get("tool_use_id") == _TOOL_CALL_ID:+ inner = block.get("content")+ assert isinstance(inner, str), inner+ return inner+ raise AssertionError(f"Anthropic tool_result {_TOOL_CALL_ID} missing")+++def _assert_task_payload_preserved(original: str, compressed: str) -> dict:+ assert "<<ccr:" not in compressed, compressed[:200]+ parsed = json.loads(compressed)+ expected = json.loads(original)+ assert parsed == expected, (sorted(expected), sorted(parsed))+ assert "tags" in parsed+ assert "subtasks" in parsed+ assert parsed["tags"] == '["cherry-pick","dedicated branch"]'+ assert len(parsed["subtasks"]) == 2+ assert parsed["subtasks"][0]["title"] == "test2"+ assert parsed["subtasks"][1]["title"] == "test4"+ return parsed+++def _wide_renamed_task_json() -> str:+ """Renamed, oversized variant so a below-threshold skip cannot hide key drop.++ Field names are arbitrary (not tags/subtasks) and expensive values sit at+ the beginning, middle, and end so a boundary/stride selector cannot+ accidentally keep them all.+ """+ original = json.loads(MCP_TASK_GET_JSON)+ expensive = {+ "nested": {"label": "child", "note": "n" * 180},+ "items": [],+ "ok": True,+ "missing": None,+ }+ wide: dict = {+ "leading_blob": expensive,+ "label_bundle": original.pop("tags"),+ "child_records": original.pop("subtasks"),+ }+ wide.update(original)+ for i in range(20):+ wide[f"extra_field_{i:02d}"] = (+ f"this is a relatively long value string for entry number {i} with content"+ )+ wide["trailing_blob"] = {+ "nested": {"label": "tail", "note": "t" * 180},+ "items": [],+ "ok": False,+ "missing": None,+ }+ payload = json.dumps(wide, separators=(",", ":"))+ assert len(payload) > 1500+ assert len(wide) > 8+ return payload+++def test_mcp_task_fixture_is_the_issue_payload() -> None:+ assert len(MCP_TASK_GET_JSON) == 813+ parsed = json.loads(MCP_TASK_GET_JSON)+ assert list(parsed)[21] == "tags"+ assert list(parsed)[22] == "subtasks"+ assert parsed["tags"] == '["cherry-pick","dedicated branch"]'+++def test_mcp_task_object_preserved_openai_tool_role() -> None:+ messages = _openai_task_messages(MCP_TASK_GET_JSON)+ result = _apply(messages)+ content = _openai_tool_content(result.messages)+ assert content == MCP_TASK_GET_JSON+ _assert_task_payload_preserved(MCP_TASK_GET_JSON, content)+ # Tool-call pairing stays intact.+ tool_msg = next(m for m in result.messages if m.get("role") == "tool")+ assistant = next(m for m in result.messages if m.get("role") == "assistant")+ assert tool_msg["tool_call_id"] == _TOOL_CALL_ID+ assert assistant["tool_calls"][0]["id"] == _TOOL_CALL_ID+ assert assistant["tool_calls"][0]["function"]["name"] == _TOOL_NAME+ assert not any("object:adaptive" in t for t in result.transforms_applied)+++def test_mcp_task_object_preserved_anthropic_tool_result() -> None:+ messages = _anthropic_task_messages(MCP_TASK_GET_JSON)+ result = _apply(messages)+ content = _anthropic_tool_content(result.messages)+ assert content == MCP_TASK_GET_JSON+ _assert_task_payload_preserved(MCP_TASK_GET_JSON, content)+ assistant = next(m for m in result.messages if m.get("role") == "assistant")+ assert assistant["content"][0]["id"] == _TOOL_CALL_ID+ user_blocks = next(+ m["content"]+ for m in result.messages+ if m.get("role") == "user" and isinstance(m.get("content"), list)+ )+ assert user_blocks[0]["tool_use_id"] == _TOOL_CALL_ID+ assert not any("object:adaptive" in t for t in result.transforms_applied)+++def test_wide_renamed_object_fields_preserved_openai_tool_role() -> None:+ payload = _wide_renamed_task_json()+ result = _apply(_openai_task_messages(payload))+ content = _openai_tool_content(result.messages)+ assert "<<ccr:" not in content+ parsed = json.loads(content)+ expected = json.loads(payload)+ assert parsed == expected+ assert "label_bundle" in parsed+ assert "child_records" in parsed+ assert "leading_blob" in parsed+ assert "trailing_blob" in parsed+ assert parsed["label_bundle"] == '["cherry-pick","dedicated branch"]'+ assert len(parsed["child_records"]) == 2+ assert parsed["leading_blob"]["nested"]["note"] == "n" * 180+ assert parsed["trailing_blob"]["nested"]["note"] == "t" * 180+++def test_malformed_json_tool_result_passes_through() -> None:+ payload = "{" + "not-json, still a tool result. " * 40+ assert len(payload) > 500+ result = _apply(_openai_task_messages(payload))+ content = _openai_tool_content(result.messages)+ assert content == payload+ assert "<<ccr:" not in content