Skip to content

armgit / codex

Public Imported from GitHub
0

Lightweight coding agent that runs in your terminal

Rust

Estimate history tokens from content instead of serialized envelopes (#45094)

Won Parkb04a2c264516
diff --git a/codex-rs/core/src/context_manager/history.rs b/codex-rs/core/src/context_manager/history.rs
index f9ad849516..fbd5a428b4 100644
--- a/codex-rs/core/src/context_manager/history.rs
+++ b/codex-rs/core/src/context_manager/history.rs
@@ -1,6 +1,7 @@
//! Parent model history and bounded host-owned context facts.
//! Compaction replaces only the model window. Snapshots include retained facts atomically;
//! checkpoint replay and source-call rollback share their live lifecycle.
+//! Token estimates charge item content rather than transport metadata.
//! Oversized instructions keep an incomplete excerpt for bounded root review, including
//! sources recovered from legacy Guardian checkpoints before their raw history is dropped.
@@ -34,6 +35,7 @@ use codex_history::ResponseItemEnvelope;
use codex_history::RetainedContext;
use codex_history::RetainedContextEvent;
use codex_history::RetainedInputSource;
+use codex_protocol::DEFAULT_FUNCTION_NAMESPACE;
use codex_protocol::items::TurnItem;
use codex_protocol::models::AgentMessageInputContent;
use codex_protocol::models::BaseInstructions;
@@ -808,8 +810,7 @@ fn estimate_encrypted_function_output_length(encoded_len: usize) -> usize {
/// Returns the same coarse, model-visible token estimate used for full history estimates.
///
-/// Ordinary items are JSON-serialized, so callers estimating many items should reuse these
-/// results instead of repeatedly estimating the full history.
+/// Counts content directly, excluding transport IDs, metadata, and outer JSON escaping.
pub(crate) fn estimate_item_token_count(item: &ResponseItem) -> i64 {
let model_visible_bytes = estimate_response_item_model_visible_bytes(item);
approx_tokens_from_byte_count_i64(model_visible_bytes)
@@ -839,6 +840,36 @@ static ORIGINAL_IMAGE_ESTIMATE_CACHE: LazyLock<BlockingLruCache<[u8; 20], Option
fn estimate_response_item_model_visible_bytes(item: &ResponseItem) -> i64 {
match item {
+ ResponseItem::Message { content, .. } => content
+ .iter()
+ .map(|part| match part {
+ ContentItem::InputText { text } | ContentItem::OutputText { text } => {
+ text_bytes(text)
+ }
+ ContentItem::InputImage { image_url, detail } => {
+ estimate_image_bytes(image_url, *detail)
+ }
+ ContentItem::InputAudio { audio_url } => estimate_audio_bytes(audio_url),
+ })
+ .fold(0i64, i64::saturating_add),
+ ResponseItem::AgentMessage {
+ author,
+ recipient,
+ content,
+ ..
+ } => content
+ .iter()
+ .map(|part| match part {
+ AgentMessageInputContent::InputText { text } => text_bytes(text),
+ AgentMessageInputContent::EncryptedContent { encrypted_content } => i64::try_from(
+ estimate_encrypted_function_output_length(encrypted_content.len()),
+ )
+ .unwrap_or(i64::MAX),
+ })
+ .fold(
+ text_bytes(author).saturating_add(text_bytes(recipient)),
+ i64::saturating_add,
+ ),
ResponseItem::Reasoning {
encrypted_content: Some(content),
..
@@ -851,46 +882,87 @@ fn estimate_response_item_model_visible_bytes(item: &ResponseItem) -> i64 {
encrypted_content: Some(content),
..
} => i64::try_from(estimate_reasoning_length(content.len())).unwrap_or(i64::MAX),
- item => {
- let raw = serialized_json_bytes(item)
- .map(|len| i64::try_from(len).unwrap_or(i64::MAX))
- .unwrap_or_default();
- let (image_payload_bytes, image_replacement_bytes) =
- image_data_url_estimate_adjustment(item);
- let (audio_payload_bytes, audio_replacement_bytes) =
- audio_data_url_estimate_adjustment(item);
- let (encrypted_payload_bytes, encrypted_replacement_bytes) =
- encrypted_function_output_estimate_adjustment(item);
- // Replace raw base64 payload bytes with per-modality estimates.
- // We intentionally preserve the data URL prefix and JSON
- // wrapper bytes already included in `raw`.
- let raw = raw
- .saturating_sub(image_payload_bytes)
- .saturating_add(image_replacement_bytes)
- .saturating_sub(audio_payload_bytes)
- .saturating_add(audio_replacement_bytes);
- raw.saturating_sub(encrypted_payload_bytes)
- .saturating_add(encrypted_replacement_bytes)
+ ResponseItem::FunctionCall {
+ name,
+ namespace,
+ arguments: input,
+ ..
+ }
+ | ResponseItem::CustomToolCall {
+ name,
+ namespace,
+ input,
+ ..
+ } => text_bytes(name)
+ .saturating_add(text_bytes(
+ namespace.as_deref().unwrap_or(DEFAULT_FUNCTION_NAMESPACE),
+ ))
+ .saturating_add(text_bytes(input)),
+ ResponseItem::FunctionCallOutput {
+ call_id,
+ name,
+ namespace,
+ output,
+ ..
+ } => estimate_function_output_bytes(&output.body)
+ .saturating_add(text_bytes(call_id.as_deref().unwrap_or_default()))
+ .saturating_add(text_bytes(name.as_deref().unwrap_or_default()))
+ .saturating_add(text_bytes(namespace.as_deref().unwrap_or_default())),
+ ResponseItem::CustomToolCallOutput {
+ call_id,
+ name,
+ output,
+ ..
+ } => estimate_function_output_bytes(&output.body)
+ .saturating_add(text_bytes(call_id))
+ .saturating_add(text_bytes(name.as_deref().unwrap_or_default())),
+ // These payloads are themselves JSON arguments, rather than transport envelopes
+ // around text. Keep their JSON syntax in the estimate.
+ ResponseItem::AdditionalTools { tools, .. } => json_content_bytes(tools),
+ ResponseItem::ToolSearchCall { arguments, .. } => json_content_bytes(arguments),
+ ResponseItem::ToolSearchOutput { tools, .. } => json_content_bytes(tools),
+ ResponseItem::LocalShellCall { action, .. } => json_content_bytes(action),
+ ResponseItem::WebSearchCall { action, .. } => {
+ action.as_ref().map(json_content_bytes).unwrap_or_default()
}
+ ResponseItem::ImageGenerationCall {
+ revised_prompt,
+ result,
+ ..
+ } => text_bytes(revised_prompt.as_deref().unwrap_or_default()).saturating_add(
+ if result.is_empty() {
+ 0
+ } else {
+ RESIZED_IMAGE_BYTES_ESTIMATE
+ },
+ ),
+ ResponseItem::ContextCompaction {
+ encrypted_content: None,
+ ..
+ } => 0,
+ // Plaintext reasoning is excluded from replay accounting.
+ ResponseItem::Reasoning {
+ encrypted_content: None,
+ ..
+ } => 0,
+ ResponseItem::ConfigurationUpdate { .. }
+ | ResponseItem::CompactionTrigger { .. }
+ | ResponseItem::Other => 0,
}
}
-/// Returns the base64 payload byte length for inline image data URLs that are
-/// eligible for token-estimation discounting.
-///
-/// We only discount payloads for `data:image/...;base64,...` URLs (case
-/// insensitive markers) and leave everything else at raw serialized size.
-fn parse_base64_image_data_url(url: &str) -> Option<&str> {
- parse_base64_data_url(url, "image/")
+fn text_bytes(text: &str) -> i64 {
+ i64::try_from(text.len()).unwrap_or(i64::MAX)
}
-/// Returns the base64 payload for inline audio data URLs that are eligible for
-/// token-estimation discounting.
-fn parse_base64_audio_data_url(url: &str) -> Option<&str> {
- parse_base64_data_url(url, "audio/")
+fn json_content_bytes(value: &(impl serde::Serialize + ?Sized)) -> i64 {
+ serialized_json_bytes(value)
+ .map(|len| i64::try_from(len).unwrap_or(i64::MAX))
+ .unwrap_or_default()
}
-fn parse_base64_data_url<'a>(url: &'a str, media_type_prefix: &str) -> Option<&'a str> {
+/// Extracts inline image bytes for the original-detail dimension estimate.
+fn parse_base64_image_data_url(url: &str) -> Option<&str> {
if !url
.get(.."data:".len())
.is_some_and(|prefix| prefix.eq_ignore_ascii_case("data:"))
@@ -908,8 +980,8 @@ fn parse_base64_data_url<'a>(url: &'a str, media_type_prefix: &str) -> Option<&'
let mime_type = metadata_parts.next().unwrap_or_default();
let has_base64_marker = metadata_parts.any(|part| part.eq_ignore_ascii_case("base64"));
if !mime_type
- .get(..media_type_prefix.len())
- .is_some_and(|prefix| prefix.eq_ignore_ascii_case(media_type_prefix))
+ .get(.."image/".len())
+ .is_some_and(|prefix| prefix.eq_ignore_ascii_case("image/"))
{
return None;
}
@@ -955,7 +1027,7 @@ fn estimate_original_image_bytes(image_url: &str) -> Option<i64> {
})
}
-/// Shared image estimate, excluding the data URL prefix and message framing.
+/// Shared image estimate, excluding message framing.
pub(crate) fn estimate_image_bytes(image_url: &str, detail: Option<ImageDetail>) -> i64 {
match detail {
Some(ImageDetail::Original) => {
@@ -965,130 +1037,35 @@ pub(crate) fn estimate_image_bytes(image_url: &str, detail: Option<ImageDetail>)
}
}
-/// Scans one response item for discount-eligible inline image data URLs and
-/// returns:
-/// - total base64 payload bytes to subtract from raw serialized size
-/// - total replacement byte estimate for those images
-fn image_data_url_estimate_adjustment(item: &ResponseItem) -> (i64, i64) {
- let mut payload_bytes = 0i64;
- let mut replacement_bytes = 0i64;
-
- let mut accumulate = |image_url: &str, detail: Option<ImageDetail>| {
- if let Some(payload_len) = parse_base64_image_data_url(image_url).map(str::len) {
- payload_bytes =
- payload_bytes.saturating_add(i64::try_from(payload_len).unwrap_or(i64::MAX));
- replacement_bytes =
- replacement_bytes.saturating_add(estimate_image_bytes(image_url, detail));
- }
- };
-
- match item {
- ResponseItem::Message { content, .. } => {
- for content_item in content {
- if let ContentItem::InputImage { image_url, detail } = content_item {
- accumulate(image_url, *detail);
- }
- }
- }
- ResponseItem::FunctionCallOutput { output, .. }
- | ResponseItem::CustomToolCallOutput { output, .. } => {
- if let FunctionCallOutputBody::ContentItems(items) = &output.body {
- for content_item in items {
- if let FunctionCallOutputContentItem::InputImage { image_url, detail } =
- content_item
- {
- accumulate(image_url, *detail);
- }
- }
- }
- }
- _ => {}
- }
-
- (payload_bytes, replacement_bytes)
+fn estimate_audio_bytes(audio_url: &str) -> i64 {
+ i64::try_from(approx_bytes_for_tokens(estimate_audio_token_count(
+ audio_url,
+ )))
+ .unwrap_or(i64::MAX)
}
-/// Scans one response item for inline base64 audio data URLs and returns:
-/// - total base64 payload bytes to subtract from raw serialized size
-/// - total replacement byte estimate for those audio inputs
-fn audio_data_url_estimate_adjustment(item: &ResponseItem) -> (i64, i64) {
- let mut payload_bytes = 0i64;
- let mut replacement_bytes = 0i64;
-
- let mut accumulate = |audio_url: &str| {
- if let Some(payload_len) = parse_base64_audio_data_url(audio_url).map(str::len) {
- payload_bytes =
- payload_bytes.saturating_add(i64::try_from(payload_len).unwrap_or(i64::MAX));
- replacement_bytes = replacement_bytes.saturating_add(
- i64::try_from(approx_bytes_for_tokens(estimate_audio_token_count(
- audio_url,
- )))
- .unwrap_or(i64::MAX),
- );
- }
- };
-
- match item {
- ResponseItem::Message { content, .. } => {
- for content_item in content {
- if let ContentItem::InputAudio { audio_url } = content_item {
- accumulate(audio_url);
+fn estimate_function_output_bytes(output: &FunctionCallOutputBody) -> i64 {
+ match output {
+ FunctionCallOutputBody::Text(text) => text_bytes(text),
+ FunctionCallOutputBody::ContentItems(items) => items
+ .iter()
+ .map(|part| match part {
+ FunctionCallOutputContentItem::InputText { text } => text_bytes(text),
+ FunctionCallOutputContentItem::InputImage { image_url, detail } => {
+ estimate_image_bytes(image_url, *detail)
}
- }
- }
- ResponseItem::FunctionCallOutput { output, .. }
- | ResponseItem::CustomToolCallOutput { output, .. } => {
- if let FunctionCallOutputBody::ContentItems(items) = &output.body {
- for content_item in items {
- if let FunctionCallOutputContentItem::InputAudio { audio_url } = content_item {
- accumulate(audio_url);
- }
+ FunctionCallOutputContentItem::InputAudio { audio_url } => {
+ estimate_audio_bytes(audio_url)
}
- }
- }
- _ => {}
- }
-
- (payload_bytes, replacement_bytes)
-}
-
-fn encrypted_function_output_estimate_adjustment(item: &ResponseItem) -> (i64, i64) {
- let mut payload_bytes = 0i64;
- let mut replacement_bytes = 0i64;
- let mut accumulate = |encrypted_content: &str| {
- payload_bytes = payload_bytes
- .saturating_add(i64::try_from(encrypted_content.len()).unwrap_or(i64::MAX));
- replacement_bytes = replacement_bytes.saturating_add(
- i64::try_from(estimate_encrypted_function_output_length(
- encrypted_content.len(),
- ))
- .unwrap_or(i64::MAX),
- );
- };
-
- match item {
- ResponseItem::FunctionCallOutput { output, .. } => {
- if let FunctionCallOutputBody::ContentItems(items) = &output.body {
- for item in items {
- if let FunctionCallOutputContentItem::EncryptedContent { encrypted_content } =
- item
- {
- accumulate(encrypted_content);
- }
- }
- }
- }
- ResponseItem::AgentMessage { content, .. } => {
- for item in content {
- if let AgentMessageInputContent::EncryptedContent { encrypted_content } = item {
- accumulate(encrypted_content);
+ FunctionCallOutputContentItem::EncryptedContent { encrypted_content } => {
+ i64::try_from(estimate_encrypted_function_output_length(
+ encrypted_content.len(),
+ ))
+ .unwrap_or(i64::MAX)
}
- }
- }
- _ => {}
+ })
+ .fold(0i64, i64::saturating_add),
}
-
- (payload_bytes, replacement_bytes)
}
fn is_model_generated_item(item: &ResponseItem) -> bool {
diff --git a/codex-rs/core/src/context_manager/history_tests.rs b/codex-rs/core/src/context_manager/history_tests.rs
index e9a2198f48..651ff3a8d7 100644
--- a/codex-rs/core/src/context_manager/history_tests.rs
+++ b/codex-rs/core/src/context_manager/history_tests.rs
@@ -2433,7 +2433,7 @@ fn image_data_url_payload_does_not_dominate_message_estimate() {
let raw_len = serde_json::to_string(&image_item).unwrap().len() as i64;
let estimated = estimate_response_item_model_visible_bytes(&image_item);
- let expected = raw_len - payload.len() as i64 + RESIZED_IMAGE_BYTES_ESTIMATE;
+ let expected = "Here is the screenshot".len() as i64 + RESIZED_IMAGE_BYTES_ESTIMATE;
let text_only_estimated = estimate_response_item_model_visible_bytes(&text_only_item);
assert_eq!(estimated, expected);
@@ -2464,7 +2464,8 @@ fn image_data_url_payload_does_not_dominate_function_call_output_estimate() {
let raw_len = serde_json::to_string(&item).unwrap().len() as i64;
let estimated = estimate_response_item_model_visible_bytes(&item);
- let expected = raw_len - payload.len() as i64 + RESIZED_IMAGE_BYTES_ESTIMATE;
+ let expected =
+ "call-abc".len() as i64 + "Screenshot captured".len() as i64 + RESIZED_IMAGE_BYTES_ESTIMATE;
assert_eq!(estimated, expected);
assert!(estimated < raw_len);
@@ -2492,7 +2493,9 @@ fn image_data_url_payload_does_not_dominate_custom_tool_call_output_estimate() {
let raw_len = serde_json::to_string(&item).unwrap().len() as i64;
let estimated = estimate_response_item_model_visible_bytes(&item);
- let expected = raw_len - payload.len() as i64 + RESIZED_IMAGE_BYTES_ESTIMATE;
+ let expected = "call-js-repl".len() as i64
+ + "Screenshot captured".len() as i64
+ + RESIZED_IMAGE_BYTES_ESTIMATE;
assert_eq!(estimated, expected);
assert!(estimated < raw_len);
@@ -2500,7 +2503,7 @@ fn image_data_url_payload_does_not_dominate_custom_tool_call_output_estimate() {
#[test]
fn audio_data_url_payload_does_not_dominate_message_estimate() {
- let (audio_url, payload_len) = pcm_wav_data_url(/*sample_count*/ 801);
+ let (audio_url, _) = pcm_wav_data_url(/*sample_count*/ 801);
let item = ResponseItem::Message {
id: None,
role: "user".to_string(),
@@ -2511,7 +2514,7 @@ fn audio_data_url_payload_does_not_dominate_message_estimate() {
let raw_len = serde_json::to_string(&item).unwrap().len() as i64;
let estimated = estimate_response_item_model_visible_bytes(&item);
- let expected = raw_len - payload_len as i64 + approx_bytes_for_tokens(/*tokens*/ 2) as i64;
+ let expected = approx_bytes_for_tokens(/*tokens*/ 2) as i64;
assert_eq!(estimated, expected);
assert!(estimated < raw_len);
@@ -2519,7 +2522,7 @@ fn audio_data_url_payload_does_not_dominate_message_estimate() {
#[test]
fn audio_data_url_payload_does_not_dominate_function_call_output_estimate() {
- let (audio_url, payload_len) = pcm_wav_data_url(/*sample_count*/ 800);
+ let (audio_url, _) = pcm_wav_data_url(/*sample_count*/ 800);
let item = ResponseItem::FunctionCallOutput {
id: None,
call_id: Some("call-audio".to_string()),
@@ -2533,7 +2536,7 @@ fn audio_data_url_payload_does_not_dominate_function_call_output_estimate() {
let raw_len = serde_json::to_string(&item).unwrap().len() as i64;
let estimated = estimate_response_item_model_visible_bytes(&item);
- let expected = raw_len - payload_len as i64 + approx_bytes_for_tokens(/*tokens*/ 1) as i64;
+ let expected = "call-audio".len() as i64 + approx_bytes_for_tokens(/*tokens*/ 1) as i64;
assert_eq!(estimated, expected);
assert!(estimated < raw_len);
@@ -2541,7 +2544,7 @@ fn audio_data_url_payload_does_not_dominate_function_call_output_estimate() {
#[test]
fn audio_data_url_payload_does_not_dominate_custom_tool_call_output_estimate() {
- let (audio_url, payload_len) = pcm_wav_data_url(/*sample_count*/ 80_000);
+ let (audio_url, _) = pcm_wav_data_url(/*sample_count*/ 80_000);
let item = ResponseItem::CustomToolCallOutput {
id: None,
call_id: "call-custom-audio".to_string(),
@@ -2554,7 +2557,8 @@ fn audio_data_url_payload_does_not_dominate_custom_tool_call_output_estimate() {
let raw_len = serde_json::to_string(&item).unwrap().len() as i64;
let estimated = estimate_response_item_model_visible_bytes(&item);
- let expected = raw_len - payload_len as i64 + approx_bytes_for_tokens(/*tokens*/ 100) as i64;
+ let expected =
+ "call-custom-audio".len() as i64 + approx_bytes_for_tokens(/*tokens*/ 100) as i64;
assert_eq!(estimated, expected);
assert!(estimated < raw_len);
@@ -2573,10 +2577,9 @@ fn malformed_audio_data_url_falls_back_to_whole_url_size_cost() {
internal_chat_message_metadata_passthrough: None,
};
- let raw_len = serde_json::to_string(&item).unwrap().len() as i64;
let estimated = estimate_response_item_model_visible_bytes(&item);
- assert_eq!(estimated, raw_len - payload.len() as i64 + fallback_bytes);
+ assert_eq!(estimated, fallback_bytes);
}
#[test]
@@ -2620,7 +2623,7 @@ fn record_items_omits_audio_that_exceeds_the_output_budget() {
}
#[test]
-fn non_base64_image_urls_are_unchanged() {
+fn non_base64_image_urls_use_image_estimates() {
let message_item = ResponseItem::Message {
id: None,
role: "user".to_string(),
@@ -2647,11 +2650,11 @@ fn non_base64_image_urls_are_unchanged() {
assert_eq!(
estimate_response_item_model_visible_bytes(&message_item),
- serde_json::to_string(&message_item).unwrap().len() as i64
+ RESIZED_IMAGE_BYTES_ESTIMATE
);
assert_eq!(
estimate_response_item_model_visible_bytes(&function_output_item),
- serde_json::to_string(&function_output_item).unwrap().len() as i64
+ "call-1".len() as i64 + RESIZED_IMAGE_BYTES_ESTIMATE
);
}
@@ -2671,9 +2674,8 @@ fn encrypted_function_output_uses_plaintext_byte_estimate() {
internal_chat_message_metadata_passthrough: None,
};
- let raw_len = serde_json::to_string(&item).unwrap().len() as i64;
let estimated = estimate_response_item_model_visible_bytes(&item);
- let expected = raw_len - encrypted_content.len() as i64
+ let expected = "call-encrypted".len() as i64
+ estimate_encrypted_function_output_length(encrypted_content.len()) as i64;
assert_eq!(estimated, expected);
@@ -2686,8 +2688,9 @@ fn encrypted_function_output_uses_plaintext_byte_estimate() {
/*trigger_turn*/ true,
)
.to_model_input_item();
- let agent_raw_len = serde_json::to_string(&agent_message).unwrap().len() as i64;
- let expected_agent = agent_raw_len - encrypted_content.len() as i64
+ let expected_agent = "/root".len() as i64
+ + "/root/worker".len() as i64
+ + "Message Type: NEW_TASK\nTask name: /root/worker\nSender: /root\nPayload:\n".len() as i64
+ estimate_encrypted_function_output_length(encrypted_content.len()) as i64;
assert_eq!(
@@ -2697,7 +2700,7 @@ fn encrypted_function_output_uses_plaintext_byte_estimate() {
}
#[test]
-fn data_url_without_base64_marker_is_unchanged() {
+fn data_url_without_base64_marker_uses_image_estimate() {
let item = ResponseItem::Message {
id: None,
role: "user".to_string(),
@@ -2711,14 +2714,15 @@ fn data_url_without_base64_marker_is_unchanged() {
assert_eq!(
estimate_response_item_model_visible_bytes(&item),
- serde_json::to_string(&item).unwrap().len() as i64
+ RESIZED_IMAGE_BYTES_ESTIMATE
);
}
#[test]
-fn non_image_base64_data_url_is_unchanged() {
+fn non_image_base64_data_url_uses_image_estimate() {
let payload = "C".repeat(4_096);
let image_url = format!("data:application/octet-stream;base64,{payload}");
+ let expected = "call-octet".len() as i64 + RESIZED_IMAGE_BYTES_ESTIMATE;
let item = ResponseItem::FunctionCallOutput {
id: None,
call_id: Some("call-octet".to_string()),
@@ -2733,10 +2737,9 @@ fn non_image_base64_data_url_is_unchanged() {
internal_chat_message_metadata_passthrough: None,
};
- let raw_len = serde_json::to_string(&item).unwrap().len() as i64;
let estimated = estimate_response_item_model_visible_bytes(&item);
- assert_eq!(estimated, raw_len);
+ assert_eq!(estimated, expected);
}
#[test]
@@ -2754,9 +2757,8 @@ fn mixed_case_data_url_markers_are_adjusted() {
internal_chat_message_metadata_passthrough: None,
};
- let raw_len = serde_json::to_string(&item).unwrap().len() as i64;
let estimated = estimate_response_item_model_visible_bytes(&item);
- let expected = raw_len - payload.len() as i64 + RESIZED_IMAGE_BYTES_ESTIMATE;
+ let expected = RESIZED_IMAGE_BYTES_ESTIMATE;
assert_eq!(estimated, expected);
}
@@ -2787,10 +2789,8 @@ fn multiple_inline_images_apply_multiple_fixed_costs() {
internal_chat_message_metadata_passthrough: None,
};
- let raw_len = serde_json::to_string(&item).unwrap().len() as i64;
- let payload_sum = (payload_one.len() + payload_two.len()) as i64;
let estimated = estimate_response_item_model_visible_bytes(&item);
- let expected = raw_len - payload_sum + (2 * RESIZED_IMAGE_BYTES_ESTIMATE);
+ let expected = "images".len() as i64 + (2 * RESIZED_IMAGE_BYTES_ESTIMATE);
assert_eq!(estimated, expected);
}
@@ -2824,9 +2824,8 @@ fn original_detail_images_scale_with_dimensions() {
internal_chat_message_metadata_passthrough: None,
};
- let raw_len = serde_json::to_string(&item).unwrap().len() as i64;
let estimated = estimate_response_item_model_visible_bytes(&item);
- let expected = raw_len - payload.len() as i64 + EXPECTED_ORIGINAL_DETAIL_IMAGE_BYTES;
+ let expected = "call-original".len() as i64 + EXPECTED_ORIGINAL_DETAIL_IMAGE_BYTES;
assert_eq!(estimated, expected);
}
@@ -2858,11 +2857,10 @@ fn original_detail_images_are_capped_at_max_patch_count() {
internal_chat_message_metadata_passthrough: None,
};
- let raw_len = serde_json::to_string(&item).unwrap().len() as i64;
let estimated = estimate_response_item_model_visible_bytes(&item);
let capped_original_detail_image_bytes =
i64::try_from(approx_bytes_for_tokens(ORIGINAL_IMAGE_MAX_PATCHES)).unwrap();
- let expected = raw_len - payload.len() as i64 + capped_original_detail_image_bytes;
+ let expected = "call-original-capped".len() as i64 + capped_original_detail_image_bytes;
assert_eq!(estimated, expected);
}
@@ -2895,15 +2893,14 @@ fn original_detail_webp_images_scale_with_dimensions() {
internal_chat_message_metadata_passthrough: None,
};
- let raw_len = serde_json::to_string(&item).unwrap().len() as i64;
let estimated = estimate_response_item_model_visible_bytes(&item);
- let expected = raw_len - payload.len() as i64 + EXPECTED_ORIGINAL_DETAIL_IMAGE_BYTES;
+ let expected = "call-original-webp".len() as i64 + EXPECTED_ORIGINAL_DETAIL_IMAGE_BYTES;
assert_eq!(estimated, expected);
}
#[test]
-fn text_only_items_unchanged() {
+fn text_only_items_count_decoded_content() {
let item = ResponseItem::Message {
id: None,
role: "assistant".to_string(),
@@ -2915,7 +2912,6 @@ fn text_only_items_unchanged() {
};
let estimated = estimate_response_item_model_visible_bytes(&item);
- let raw_len = serde_json::to_string(&item).unwrap().len() as i64;
- assert_eq!(estimated, raw_len);
+ assert_eq!(estimated, "Hello, \"world\"!\nこんにちは".len() as i64);
}
diff --git a/codex-rs/core/tests/suite/compact_remote_trimming.rs b/codex-rs/core/tests/suite/compact_remote_trimming.rs
index 04ed0f552a..912bf9f6a1 100644
--- a/codex-rs/core/tests/suite/compact_remote_trimming.rs
+++ b/codex-rs/core/tests/suite/compact_remote_trimming.rs
@@ -14,6 +14,78 @@ fn compact_response() -> String {
])
}
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn remote_compact_v2_token_estimate_ignores_message_bookkeeping_and_json_escaping()
+-> Result<()> {
+ skip_if_no_network!(Ok(()));
+
+ let harness = TestCodexHarness::with_auto_env_builder(
+ test_codex().with_auth(CodexAuth::create_dummy_chatgpt_auth_for_testing()),
+ )
+ .await?;
+ let escaped_text = "a \"quoted\" line\nand\\path";
+ let plain_text = "x".repeat(escaped_text.len());
+ let mut estimates = Vec::new();
+ for (id, text, metadata) in [
+ (
+ "msg_1",
+ plain_text.as_str(),
+ json!({
+ "turn_id": "turn-1",
+ "create_time": 1,
+ "content_item_kinds": ["unknown"],
+ }),
+ ),
+ (
+ "msg_019f0000-0000-7000-8000-000000000001",
+ escaped_text,
+ json!({
+ "turn_id": "019f0000-0000-7000-8000-000000000002",
+ "create_time": 1_700_000_000.123,
+ "content_item_kinds": ["user.text"],
+ }),
+ ),
+ ] {
+ let codex = harness
+ .test()
+ .thread_manager
+ .start_thread(StartThreadOptions {
+ environments: Some(vec![
+ harness.test().executor_environment().selection().clone(),
+ ]),
+ ..StartThreadOptions::new(harness.test().config.clone())
+ })
+ .await?
+ .thread;
+ let message = json!({
+ "type": "message",
+ "id": id,
+ "role": "user",
+ "content": [{"type": "input_text", "text": text}],
+ "internal_chat_message_metadata_passthrough": metadata,
+ });
+ codex
+ .inject_response_items(vec![serde_json::from_value(message.clone())?])
+ .await?;
+ let mock = mount_sse_once(harness.server(), compact_response()).await;
+ codex.submit(Op::Compact).await?;
+ wait_for_turn_complete(&codex).await;
+
+ estimates.push(codex.token_usage_info().await.context("token usage")?);
+ assert_eq!(
+ mock.single_request()
+ .input()
+ .into_iter()
+ .find(|item| item["id"] == id),
+ Some(message),
+ );
+ codex.shutdown_and_wait().await?;
+ }
+
+ assert_eq!(estimates[0], estimates[1]);
+ Ok(())
+}
+
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn remote_compact_v2_trims_tool_search_output_to_empty_tools_array() -> Result<()> {
skip_if_no_network!(Ok(()));