Lightweight coding agent that runs in your terminal
Rust
Estimate history tokens from content instead of serialized envelopes (#45094)
Won Parkb04a2c264516
diff --git a/codex-rs/core/src/context_manager/history.rs b/codex-rs/core/src/context_manager/history.rsindex f9ad849516..fbd5a428b4 100644--- a/codex-rs/core/src/context_manager/history.rs+++ b/codex-rs/core/src/context_manager/history.rs@@ -1,6 +1,7 @@//! Parent model history and bounded host-owned context facts.//! Compaction replaces only the model window. Snapshots include retained facts atomically;//! checkpoint replay and source-call rollback share their live lifecycle.+//! Token estimates charge item content rather than transport metadata.//! Oversized instructions keep an incomplete excerpt for bounded root review, including//! sources recovered from legacy Guardian checkpoints before their raw history is dropped.@@ -34,6 +35,7 @@ use codex_history::ResponseItemEnvelope;use codex_history::RetainedContext;use codex_history::RetainedContextEvent;use codex_history::RetainedInputSource;+use codex_protocol::DEFAULT_FUNCTION_NAMESPACE;use codex_protocol::items::TurnItem;use codex_protocol::models::AgentMessageInputContent;use codex_protocol::models::BaseInstructions;@@ -808,8 +810,7 @@ fn estimate_encrypted_function_output_length(encoded_len: usize) -> usize {/// Returns the same coarse, model-visible token estimate used for full history estimates.///-/// Ordinary items are JSON-serialized, so callers estimating many items should reuse these-/// results instead of repeatedly estimating the full history.+/// Counts content directly, excluding transport IDs, metadata, and outer JSON escaping.pub(crate) fn estimate_item_token_count(item: &ResponseItem) -> i64 {let model_visible_bytes = estimate_response_item_model_visible_bytes(item);approx_tokens_from_byte_count_i64(model_visible_bytes)@@ -839,6 +840,36 @@ static ORIGINAL_IMAGE_ESTIMATE_CACHE: LazyLock<BlockingLruCache<[u8; 20], Optionfn estimate_response_item_model_visible_bytes(item: &ResponseItem) -> i64 {match item {+ ResponseItem::Message { content, .. } => content+ .iter()+ .map(|part| match part {+ ContentItem::InputText { text } | ContentItem::OutputText { text } => {+ text_bytes(text)+ }+ ContentItem::InputImage { image_url, detail } => {+ estimate_image_bytes(image_url, *detail)+ }+ ContentItem::InputAudio { audio_url } => estimate_audio_bytes(audio_url),+ })+ .fold(0i64, i64::saturating_add),+ ResponseItem::AgentMessage {+ author,+ recipient,+ content,+ ..+ } => content+ .iter()+ .map(|part| match part {+ AgentMessageInputContent::InputText { text } => text_bytes(text),+ AgentMessageInputContent::EncryptedContent { encrypted_content } => i64::try_from(+ estimate_encrypted_function_output_length(encrypted_content.len()),+ )+ .unwrap_or(i64::MAX),+ })+ .fold(+ text_bytes(author).saturating_add(text_bytes(recipient)),+ i64::saturating_add,+ ),ResponseItem::Reasoning {encrypted_content: Some(content),..@@ -851,46 +882,87 @@ fn estimate_response_item_model_visible_bytes(item: &ResponseItem) -> i64 {encrypted_content: Some(content),..} => i64::try_from(estimate_reasoning_length(content.len())).unwrap_or(i64::MAX),- item => {- let raw = serialized_json_bytes(item)- .map(|len| i64::try_from(len).unwrap_or(i64::MAX))- .unwrap_or_default();- let (image_payload_bytes, image_replacement_bytes) =- image_data_url_estimate_adjustment(item);- let (audio_payload_bytes, audio_replacement_bytes) =- audio_data_url_estimate_adjustment(item);- let (encrypted_payload_bytes, encrypted_replacement_bytes) =- encrypted_function_output_estimate_adjustment(item);- // Replace raw base64 payload bytes with per-modality estimates.- // We intentionally preserve the data URL prefix and JSON- // wrapper bytes already included in `raw`.- let raw = raw- .saturating_sub(image_payload_bytes)- .saturating_add(image_replacement_bytes)- .saturating_sub(audio_payload_bytes)- .saturating_add(audio_replacement_bytes);- raw.saturating_sub(encrypted_payload_bytes)- .saturating_add(encrypted_replacement_bytes)+ ResponseItem::FunctionCall {+ name,+ namespace,+ arguments: input,+ ..+ }+ | ResponseItem::CustomToolCall {+ name,+ namespace,+ input,+ ..+ } => text_bytes(name)+ .saturating_add(text_bytes(+ namespace.as_deref().unwrap_or(DEFAULT_FUNCTION_NAMESPACE),+ ))+ .saturating_add(text_bytes(input)),+ ResponseItem::FunctionCallOutput {+ call_id,+ name,+ namespace,+ output,+ ..+ } => estimate_function_output_bytes(&output.body)+ .saturating_add(text_bytes(call_id.as_deref().unwrap_or_default()))+ .saturating_add(text_bytes(name.as_deref().unwrap_or_default()))+ .saturating_add(text_bytes(namespace.as_deref().unwrap_or_default())),+ ResponseItem::CustomToolCallOutput {+ call_id,+ name,+ output,+ ..+ } => estimate_function_output_bytes(&output.body)+ .saturating_add(text_bytes(call_id))+ .saturating_add(text_bytes(name.as_deref().unwrap_or_default())),+ // These payloads are themselves JSON arguments, rather than transport envelopes+ // around text. Keep their JSON syntax in the estimate.+ ResponseItem::AdditionalTools { tools, .. } => json_content_bytes(tools),+ ResponseItem::ToolSearchCall { arguments, .. } => json_content_bytes(arguments),+ ResponseItem::ToolSearchOutput { tools, .. } => json_content_bytes(tools),+ ResponseItem::LocalShellCall { action, .. } => json_content_bytes(action),+ ResponseItem::WebSearchCall { action, .. } => {+ action.as_ref().map(json_content_bytes).unwrap_or_default()}+ ResponseItem::ImageGenerationCall {+ revised_prompt,+ result,+ ..+ } => text_bytes(revised_prompt.as_deref().unwrap_or_default()).saturating_add(+ if result.is_empty() {+ 0+ } else {+ RESIZED_IMAGE_BYTES_ESTIMATE+ },+ ),+ ResponseItem::ContextCompaction {+ encrypted_content: None,+ ..+ } => 0,+ // Plaintext reasoning is excluded from replay accounting.+ ResponseItem::Reasoning {+ encrypted_content: None,+ ..+ } => 0,+ ResponseItem::ConfigurationUpdate { .. }+ | ResponseItem::CompactionTrigger { .. }+ | ResponseItem::Other => 0,}}-/// Returns the base64 payload byte length for inline image data URLs that are-/// eligible for token-estimation discounting.-///-/// We only discount payloads for `data:image/...;base64,...` URLs (case-/// insensitive markers) and leave everything else at raw serialized size.-fn parse_base64_image_data_url(url: &str) -> Option<&str> {- parse_base64_data_url(url, "image/")+fn text_bytes(text: &str) -> i64 {+ i64::try_from(text.len()).unwrap_or(i64::MAX)}-/// Returns the base64 payload for inline audio data URLs that are eligible for-/// token-estimation discounting.-fn parse_base64_audio_data_url(url: &str) -> Option<&str> {- parse_base64_data_url(url, "audio/")+fn json_content_bytes(value: &(impl serde::Serialize + ?Sized)) -> i64 {+ serialized_json_bytes(value)+ .map(|len| i64::try_from(len).unwrap_or(i64::MAX))+ .unwrap_or_default()}-fn parse_base64_data_url<'a>(url: &'a str, media_type_prefix: &str) -> Option<&'a str> {+/// Extracts inline image bytes for the original-detail dimension estimate.+fn parse_base64_image_data_url(url: &str) -> Option<&str> {if !url.get(.."data:".len()).is_some_and(|prefix| prefix.eq_ignore_ascii_case("data:"))@@ -908,8 +980,8 @@ fn parse_base64_data_url<'a>(url: &'a str, media_type_prefix: &str) -> Option<&'let mime_type = metadata_parts.next().unwrap_or_default();let has_base64_marker = metadata_parts.any(|part| part.eq_ignore_ascii_case("base64"));if !mime_type- .get(..media_type_prefix.len())- .is_some_and(|prefix| prefix.eq_ignore_ascii_case(media_type_prefix))+ .get(.."image/".len())+ .is_some_and(|prefix| prefix.eq_ignore_ascii_case("image/")){return None;}@@ -955,7 +1027,7 @@ fn estimate_original_image_bytes(image_url: &str) -> Option<i64> {})}-/// Shared image estimate, excluding the data URL prefix and message framing.+/// Shared image estimate, excluding message framing.pub(crate) fn estimate_image_bytes(image_url: &str, detail: Option<ImageDetail>) -> i64 {match detail {Some(ImageDetail::Original) => {@@ -965,130 +1037,35 @@ pub(crate) fn estimate_image_bytes(image_url: &str, detail: Option<ImageDetail>)}}-/// Scans one response item for discount-eligible inline image data URLs and-/// returns:-/// - total base64 payload bytes to subtract from raw serialized size-/// - total replacement byte estimate for those images-fn image_data_url_estimate_adjustment(item: &ResponseItem) -> (i64, i64) {- let mut payload_bytes = 0i64;- let mut replacement_bytes = 0i64;-- let mut accumulate = |image_url: &str, detail: Option<ImageDetail>| {- if let Some(payload_len) = parse_base64_image_data_url(image_url).map(str::len) {- payload_bytes =- payload_bytes.saturating_add(i64::try_from(payload_len).unwrap_or(i64::MAX));- replacement_bytes =- replacement_bytes.saturating_add(estimate_image_bytes(image_url, detail));- }- };-- match item {- ResponseItem::Message { content, .. } => {- for content_item in content {- if let ContentItem::InputImage { image_url, detail } = content_item {- accumulate(image_url, *detail);- }- }- }- ResponseItem::FunctionCallOutput { output, .. }- | ResponseItem::CustomToolCallOutput { output, .. } => {- if let FunctionCallOutputBody::ContentItems(items) = &output.body {- for content_item in items {- if let FunctionCallOutputContentItem::InputImage { image_url, detail } =- content_item- {- accumulate(image_url, *detail);- }- }- }- }- _ => {}- }-- (payload_bytes, replacement_bytes)+fn estimate_audio_bytes(audio_url: &str) -> i64 {+ i64::try_from(approx_bytes_for_tokens(estimate_audio_token_count(+ audio_url,+ )))+ .unwrap_or(i64::MAX)}-/// Scans one response item for inline base64 audio data URLs and returns:-/// - total base64 payload bytes to subtract from raw serialized size-/// - total replacement byte estimate for those audio inputs-fn audio_data_url_estimate_adjustment(item: &ResponseItem) -> (i64, i64) {- let mut payload_bytes = 0i64;- let mut replacement_bytes = 0i64;-- let mut accumulate = |audio_url: &str| {- if let Some(payload_len) = parse_base64_audio_data_url(audio_url).map(str::len) {- payload_bytes =- payload_bytes.saturating_add(i64::try_from(payload_len).unwrap_or(i64::MAX));- replacement_bytes = replacement_bytes.saturating_add(- i64::try_from(approx_bytes_for_tokens(estimate_audio_token_count(- audio_url,- )))- .unwrap_or(i64::MAX),- );- }- };-- match item {- ResponseItem::Message { content, .. } => {- for content_item in content {- if let ContentItem::InputAudio { audio_url } = content_item {- accumulate(audio_url);+fn estimate_function_output_bytes(output: &FunctionCallOutputBody) -> i64 {+ match output {+ FunctionCallOutputBody::Text(text) => text_bytes(text),+ FunctionCallOutputBody::ContentItems(items) => items+ .iter()+ .map(|part| match part {+ FunctionCallOutputContentItem::InputText { text } => text_bytes(text),+ FunctionCallOutputContentItem::InputImage { image_url, detail } => {+ estimate_image_bytes(image_url, *detail)}- }- }- ResponseItem::FunctionCallOutput { output, .. }- | ResponseItem::CustomToolCallOutput { output, .. } => {- if let FunctionCallOutputBody::ContentItems(items) = &output.body {- for content_item in items {- if let FunctionCallOutputContentItem::InputAudio { audio_url } = content_item {- accumulate(audio_url);- }+ FunctionCallOutputContentItem::InputAudio { audio_url } => {+ estimate_audio_bytes(audio_url)}- }- }- _ => {}- }-- (payload_bytes, replacement_bytes)-}--fn encrypted_function_output_estimate_adjustment(item: &ResponseItem) -> (i64, i64) {- let mut payload_bytes = 0i64;- let mut replacement_bytes = 0i64;- let mut accumulate = |encrypted_content: &str| {- payload_bytes = payload_bytes- .saturating_add(i64::try_from(encrypted_content.len()).unwrap_or(i64::MAX));- replacement_bytes = replacement_bytes.saturating_add(- i64::try_from(estimate_encrypted_function_output_length(- encrypted_content.len(),- ))- .unwrap_or(i64::MAX),- );- };-- match item {- ResponseItem::FunctionCallOutput { output, .. } => {- if let FunctionCallOutputBody::ContentItems(items) = &output.body {- for item in items {- if let FunctionCallOutputContentItem::EncryptedContent { encrypted_content } =- item- {- accumulate(encrypted_content);- }- }- }- }- ResponseItem::AgentMessage { content, .. } => {- for item in content {- if let AgentMessageInputContent::EncryptedContent { encrypted_content } = item {- accumulate(encrypted_content);+ FunctionCallOutputContentItem::EncryptedContent { encrypted_content } => {+ i64::try_from(estimate_encrypted_function_output_length(+ encrypted_content.len(),+ ))+ .unwrap_or(i64::MAX)}- }- }- _ => {}+ })+ .fold(0i64, i64::saturating_add),}-- (payload_bytes, replacement_bytes)}fn is_model_generated_item(item: &ResponseItem) -> bool {diff --git a/codex-rs/core/src/context_manager/history_tests.rs b/codex-rs/core/src/context_manager/history_tests.rsindex e9a2198f48..651ff3a8d7 100644--- a/codex-rs/core/src/context_manager/history_tests.rs+++ b/codex-rs/core/src/context_manager/history_tests.rs@@ -2433,7 +2433,7 @@ fn image_data_url_payload_does_not_dominate_message_estimate() {let raw_len = serde_json::to_string(&image_item).unwrap().len() as i64;let estimated = estimate_response_item_model_visible_bytes(&image_item);- let expected = raw_len - payload.len() as i64 + RESIZED_IMAGE_BYTES_ESTIMATE;+ let expected = "Here is the screenshot".len() as i64 + RESIZED_IMAGE_BYTES_ESTIMATE;let text_only_estimated = estimate_response_item_model_visible_bytes(&text_only_item);assert_eq!(estimated, expected);@@ -2464,7 +2464,8 @@ fn image_data_url_payload_does_not_dominate_function_call_output_estimate() {let raw_len = serde_json::to_string(&item).unwrap().len() as i64;let estimated = estimate_response_item_model_visible_bytes(&item);- let expected = raw_len - payload.len() as i64 + RESIZED_IMAGE_BYTES_ESTIMATE;+ let expected =+ "call-abc".len() as i64 + "Screenshot captured".len() as i64 + RESIZED_IMAGE_BYTES_ESTIMATE;assert_eq!(estimated, expected);assert!(estimated < raw_len);@@ -2492,7 +2493,9 @@ fn image_data_url_payload_does_not_dominate_custom_tool_call_output_estimate() {let raw_len = serde_json::to_string(&item).unwrap().len() as i64;let estimated = estimate_response_item_model_visible_bytes(&item);- let expected = raw_len - payload.len() as i64 + RESIZED_IMAGE_BYTES_ESTIMATE;+ let expected = "call-js-repl".len() as i64+ + "Screenshot captured".len() as i64+ + RESIZED_IMAGE_BYTES_ESTIMATE;assert_eq!(estimated, expected);assert!(estimated < raw_len);@@ -2500,7 +2503,7 @@ fn image_data_url_payload_does_not_dominate_custom_tool_call_output_estimate() {#[test]fn audio_data_url_payload_does_not_dominate_message_estimate() {- let (audio_url, payload_len) = pcm_wav_data_url(/*sample_count*/ 801);+ let (audio_url, _) = pcm_wav_data_url(/*sample_count*/ 801);let item = ResponseItem::Message {id: None,role: "user".to_string(),@@ -2511,7 +2514,7 @@ fn audio_data_url_payload_does_not_dominate_message_estimate() {let raw_len = serde_json::to_string(&item).unwrap().len() as i64;let estimated = estimate_response_item_model_visible_bytes(&item);- let expected = raw_len - payload_len as i64 + approx_bytes_for_tokens(/*tokens*/ 2) as i64;+ let expected = approx_bytes_for_tokens(/*tokens*/ 2) as i64;assert_eq!(estimated, expected);assert!(estimated < raw_len);@@ -2519,7 +2522,7 @@ fn audio_data_url_payload_does_not_dominate_message_estimate() {#[test]fn audio_data_url_payload_does_not_dominate_function_call_output_estimate() {- let (audio_url, payload_len) = pcm_wav_data_url(/*sample_count*/ 800);+ let (audio_url, _) = pcm_wav_data_url(/*sample_count*/ 800);let item = ResponseItem::FunctionCallOutput {id: None,call_id: Some("call-audio".to_string()),@@ -2533,7 +2536,7 @@ fn audio_data_url_payload_does_not_dominate_function_call_output_estimate() {let raw_len = serde_json::to_string(&item).unwrap().len() as i64;let estimated = estimate_response_item_model_visible_bytes(&item);- let expected = raw_len - payload_len as i64 + approx_bytes_for_tokens(/*tokens*/ 1) as i64;+ let expected = "call-audio".len() as i64 + approx_bytes_for_tokens(/*tokens*/ 1) as i64;assert_eq!(estimated, expected);assert!(estimated < raw_len);@@ -2541,7 +2544,7 @@ fn audio_data_url_payload_does_not_dominate_function_call_output_estimate() {#[test]fn audio_data_url_payload_does_not_dominate_custom_tool_call_output_estimate() {- let (audio_url, payload_len) = pcm_wav_data_url(/*sample_count*/ 80_000);+ let (audio_url, _) = pcm_wav_data_url(/*sample_count*/ 80_000);let item = ResponseItem::CustomToolCallOutput {id: None,call_id: "call-custom-audio".to_string(),@@ -2554,7 +2557,8 @@ fn audio_data_url_payload_does_not_dominate_custom_tool_call_output_estimate() {let raw_len = serde_json::to_string(&item).unwrap().len() as i64;let estimated = estimate_response_item_model_visible_bytes(&item);- let expected = raw_len - payload_len as i64 + approx_bytes_for_tokens(/*tokens*/ 100) as i64;+ let expected =+ "call-custom-audio".len() as i64 + approx_bytes_for_tokens(/*tokens*/ 100) as i64;assert_eq!(estimated, expected);assert!(estimated < raw_len);@@ -2573,10 +2577,9 @@ fn malformed_audio_data_url_falls_back_to_whole_url_size_cost() {internal_chat_message_metadata_passthrough: None,};- let raw_len = serde_json::to_string(&item).unwrap().len() as i64;let estimated = estimate_response_item_model_visible_bytes(&item);- assert_eq!(estimated, raw_len - payload.len() as i64 + fallback_bytes);+ assert_eq!(estimated, fallback_bytes);}#[test]@@ -2620,7 +2623,7 @@ fn record_items_omits_audio_that_exceeds_the_output_budget() {}#[test]-fn non_base64_image_urls_are_unchanged() {+fn non_base64_image_urls_use_image_estimates() {let message_item = ResponseItem::Message {id: None,role: "user".to_string(),@@ -2647,11 +2650,11 @@ fn non_base64_image_urls_are_unchanged() {assert_eq!(estimate_response_item_model_visible_bytes(&message_item),- serde_json::to_string(&message_item).unwrap().len() as i64+ RESIZED_IMAGE_BYTES_ESTIMATE);assert_eq!(estimate_response_item_model_visible_bytes(&function_output_item),- serde_json::to_string(&function_output_item).unwrap().len() as i64+ "call-1".len() as i64 + RESIZED_IMAGE_BYTES_ESTIMATE);}@@ -2671,9 +2674,8 @@ fn encrypted_function_output_uses_plaintext_byte_estimate() {internal_chat_message_metadata_passthrough: None,};- let raw_len = serde_json::to_string(&item).unwrap().len() as i64;let estimated = estimate_response_item_model_visible_bytes(&item);- let expected = raw_len - encrypted_content.len() as i64+ let expected = "call-encrypted".len() as i64+ estimate_encrypted_function_output_length(encrypted_content.len()) as i64;assert_eq!(estimated, expected);@@ -2686,8 +2688,9 @@ fn encrypted_function_output_uses_plaintext_byte_estimate() {/*trigger_turn*/ true,).to_model_input_item();- let agent_raw_len = serde_json::to_string(&agent_message).unwrap().len() as i64;- let expected_agent = agent_raw_len - encrypted_content.len() as i64+ let expected_agent = "/root".len() as i64+ + "/root/worker".len() as i64+ + "Message Type: NEW_TASK\nTask name: /root/worker\nSender: /root\nPayload:\n".len() as i64+ estimate_encrypted_function_output_length(encrypted_content.len()) as i64;assert_eq!(@@ -2697,7 +2700,7 @@ fn encrypted_function_output_uses_plaintext_byte_estimate() {}#[test]-fn data_url_without_base64_marker_is_unchanged() {+fn data_url_without_base64_marker_uses_image_estimate() {let item = ResponseItem::Message {id: None,role: "user".to_string(),@@ -2711,14 +2714,15 @@ fn data_url_without_base64_marker_is_unchanged() {assert_eq!(estimate_response_item_model_visible_bytes(&item),- serde_json::to_string(&item).unwrap().len() as i64+ RESIZED_IMAGE_BYTES_ESTIMATE);}#[test]-fn non_image_base64_data_url_is_unchanged() {+fn non_image_base64_data_url_uses_image_estimate() {let payload = "C".repeat(4_096);let image_url = format!("data:application/octet-stream;base64,{payload}");+ let expected = "call-octet".len() as i64 + RESIZED_IMAGE_BYTES_ESTIMATE;let item = ResponseItem::FunctionCallOutput {id: None,call_id: Some("call-octet".to_string()),@@ -2733,10 +2737,9 @@ fn non_image_base64_data_url_is_unchanged() {internal_chat_message_metadata_passthrough: None,};- let raw_len = serde_json::to_string(&item).unwrap().len() as i64;let estimated = estimate_response_item_model_visible_bytes(&item);- assert_eq!(estimated, raw_len);+ assert_eq!(estimated, expected);}#[test]@@ -2754,9 +2757,8 @@ fn mixed_case_data_url_markers_are_adjusted() {internal_chat_message_metadata_passthrough: None,};- let raw_len = serde_json::to_string(&item).unwrap().len() as i64;let estimated = estimate_response_item_model_visible_bytes(&item);- let expected = raw_len - payload.len() as i64 + RESIZED_IMAGE_BYTES_ESTIMATE;+ let expected = RESIZED_IMAGE_BYTES_ESTIMATE;assert_eq!(estimated, expected);}@@ -2787,10 +2789,8 @@ fn multiple_inline_images_apply_multiple_fixed_costs() {internal_chat_message_metadata_passthrough: None,};- let raw_len = serde_json::to_string(&item).unwrap().len() as i64;- let payload_sum = (payload_one.len() + payload_two.len()) as i64;let estimated = estimate_response_item_model_visible_bytes(&item);- let expected = raw_len - payload_sum + (2 * RESIZED_IMAGE_BYTES_ESTIMATE);+ let expected = "images".len() as i64 + (2 * RESIZED_IMAGE_BYTES_ESTIMATE);assert_eq!(estimated, expected);}@@ -2824,9 +2824,8 @@ fn original_detail_images_scale_with_dimensions() {internal_chat_message_metadata_passthrough: None,};- let raw_len = serde_json::to_string(&item).unwrap().len() as i64;let estimated = estimate_response_item_model_visible_bytes(&item);- let expected = raw_len - payload.len() as i64 + EXPECTED_ORIGINAL_DETAIL_IMAGE_BYTES;+ let expected = "call-original".len() as i64 + EXPECTED_ORIGINAL_DETAIL_IMAGE_BYTES;assert_eq!(estimated, expected);}@@ -2858,11 +2857,10 @@ fn original_detail_images_are_capped_at_max_patch_count() {internal_chat_message_metadata_passthrough: None,};- let raw_len = serde_json::to_string(&item).unwrap().len() as i64;let estimated = estimate_response_item_model_visible_bytes(&item);let capped_original_detail_image_bytes =i64::try_from(approx_bytes_for_tokens(ORIGINAL_IMAGE_MAX_PATCHES)).unwrap();- let expected = raw_len - payload.len() as i64 + capped_original_detail_image_bytes;+ let expected = "call-original-capped".len() as i64 + capped_original_detail_image_bytes;assert_eq!(estimated, expected);}@@ -2895,15 +2893,14 @@ fn original_detail_webp_images_scale_with_dimensions() {internal_chat_message_metadata_passthrough: None,};- let raw_len = serde_json::to_string(&item).unwrap().len() as i64;let estimated = estimate_response_item_model_visible_bytes(&item);- let expected = raw_len - payload.len() as i64 + EXPECTED_ORIGINAL_DETAIL_IMAGE_BYTES;+ let expected = "call-original-webp".len() as i64 + EXPECTED_ORIGINAL_DETAIL_IMAGE_BYTES;assert_eq!(estimated, expected);}#[test]-fn text_only_items_unchanged() {+fn text_only_items_count_decoded_content() {let item = ResponseItem::Message {id: None,role: "assistant".to_string(),@@ -2915,7 +2912,6 @@ fn text_only_items_unchanged() {};let estimated = estimate_response_item_model_visible_bytes(&item);- let raw_len = serde_json::to_string(&item).unwrap().len() as i64;- assert_eq!(estimated, raw_len);+ assert_eq!(estimated, "Hello, \"world\"!\nこんにちは".len() as i64);}diff --git a/codex-rs/core/tests/suite/compact_remote_trimming.rs b/codex-rs/core/tests/suite/compact_remote_trimming.rsindex 04ed0f552a..912bf9f6a1 100644--- a/codex-rs/core/tests/suite/compact_remote_trimming.rs+++ b/codex-rs/core/tests/suite/compact_remote_trimming.rs@@ -14,6 +14,78 @@ fn compact_response() -> String {])}+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]+async fn remote_compact_v2_token_estimate_ignores_message_bookkeeping_and_json_escaping()+-> Result<()> {+ skip_if_no_network!(Ok(()));++ let harness = TestCodexHarness::with_auto_env_builder(+ test_codex().with_auth(CodexAuth::create_dummy_chatgpt_auth_for_testing()),+ )+ .await?;+ let escaped_text = "a \"quoted\" line\nand\\path";+ let plain_text = "x".repeat(escaped_text.len());+ let mut estimates = Vec::new();+ for (id, text, metadata) in [+ (+ "msg_1",+ plain_text.as_str(),+ json!({+ "turn_id": "turn-1",+ "create_time": 1,+ "content_item_kinds": ["unknown"],+ }),+ ),+ (+ "msg_019f0000-0000-7000-8000-000000000001",+ escaped_text,+ json!({+ "turn_id": "019f0000-0000-7000-8000-000000000002",+ "create_time": 1_700_000_000.123,+ "content_item_kinds": ["user.text"],+ }),+ ),+ ] {+ let codex = harness+ .test()+ .thread_manager+ .start_thread(StartThreadOptions {+ environments: Some(vec![+ harness.test().executor_environment().selection().clone(),+ ]),+ ..StartThreadOptions::new(harness.test().config.clone())+ })+ .await?+ .thread;+ let message = json!({+ "type": "message",+ "id": id,+ "role": "user",+ "content": [{"type": "input_text", "text": text}],+ "internal_chat_message_metadata_passthrough": metadata,+ });+ codex+ .inject_response_items(vec![serde_json::from_value(message.clone())?])+ .await?;+ let mock = mount_sse_once(harness.server(), compact_response()).await;+ codex.submit(Op::Compact).await?;+ wait_for_turn_complete(&codex).await;++ estimates.push(codex.token_usage_info().await.context("token usage")?);+ assert_eq!(+ mock.single_request()+ .input()+ .into_iter()+ .find(|item| item["id"] == id),+ Some(message),+ );+ codex.shutdown_and_wait().await?;+ }++ assert_eq!(estimates[0], estimates[1]);+ Ok(())+}+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]async fn remote_compact_v2_trims_tool_search_output_to_empty_tools_array() -> Result<()> {skip_if_no_network!(Ok(()));