Lightweight coding agent that runs in your terminal
Rust
Preserve annotations when omitting unsupported media (#40277)
pakrym-oaiaec653daa987
diff --git a/codex-rs/core/src/context/mod.rs b/codex-rs/core/src/context/mod.rsindex a2a68544e7..378fe0cb08 100644--- a/codex-rs/core/src/context/mod.rs+++ b/codex-rs/core/src/context/mod.rs@@ -38,6 +38,7 @@ mod rollout_budget;mod subagent_notification;mod token_budget_context;mod turn_aborted;+mod unsupported_media;mod user_instructions;mod user_shell_command;pub(crate) mod world_state;@@ -98,6 +99,7 @@ pub(crate) use token_budget_context::TokenBudgetContext;pub(crate) use token_budget_context::TokenBudgetRemainingContext;pub(crate) use token_budget_context::TokenBudgetReminder;pub(crate) use turn_aborted::TurnAborted;+pub(crate) use unsupported_media::UnsupportedMedia;pub(crate) use user_instructions::UserInstructions;pub(crate) use user_shell_command::UserShellCommand;pub(crate) use world_state::ManagedDeveloperInstructions;diff --git a/codex-rs/core/src/context/unsupported_media.rs b/codex-rs/core/src/context/unsupported_media.rsnew file mode 100644index 0000000000..8b225649c0--- /dev/null+++ b/codex-rs/core/src/context/unsupported_media.rs@@ -0,0 +1,42 @@+use super::ContextualUserFragment;+use codex_protocol::models::ContentItemKind;++#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub(crate) struct UnsupportedMedia {+ text: &'static str,+ content_kind: &'static str,+}++impl UnsupportedMedia {+ pub(crate) const IMAGE: Self = Self {+ text: "image content omitted because you do not support image input",+ content_kind: "images.unsupported",+ };++ pub(crate) const AUDIO: Self = Self {+ text: "audio content omitted because you do not support audio input",+ content_kind: "audio.unsupported",+ };+}++impl ContextualUserFragment for UnsupportedMedia {+ fn content_kind(&self) -> ContentItemKind {+ ContentItemKind(self.content_kind.to_string())+ }++ fn role(&self) -> &'static str {+ "user"+ }++ fn markers(&self) -> (&'static str, &'static str) {+ Self::type_markers()+ }++ fn type_markers() -> (&'static str, &'static str) {+ ("", "")+ }++ fn body(&self) -> String {+ self.text.to_string()+ }+}diff --git a/codex-rs/core/src/context_manager/history_tests.rs b/codex-rs/core/src/context_manager/history_tests.rsindex 5a0c6e2602..06fa8b4b08 100644--- a/codex-rs/core/src/context_manager/history_tests.rs+++ b/codex-rs/core/src/context_manager/history_tests.rs@@ -46,6 +46,13 @@ const EXEC_FORMAT_MAX_BYTES: usize = 10_000;const EXEC_FORMAT_MAX_TOKENS: usize = 2_500;const TEST_WAV_SAMPLE_RATE: u32 = 8_000;+fn unknown_content_metadata() -> InternalChatMessageMetadataPassthrough {+ InternalChatMessageMetadataPassthrough {+ content_item_kinds: Some(vec![ContentItemKind("unknown".to_string())]),+ ..Default::default()+ }+}+fn pcm_wav_data_url(sample_count: u32) -> (String, usize) {let padding = sample_count % 2;let mut bytes = Vec::new();@@ -78,7 +85,7 @@ fn assistant_msg(text: &str) -> ResponseItem {text: text.to_string(),}],phase: None,- internal_chat_message_metadata_passthrough: None,+ internal_chat_message_metadata_passthrough: Some(unknown_content_metadata()),}}@@ -97,7 +104,7 @@ fn inter_agent_assistant_msg(text: &str) -> ResponseItem {text: serde_json::to_string(&communication).unwrap(),}],phase: None,- internal_chat_message_metadata_passthrough: None,+ internal_chat_message_metadata_passthrough: Some(unknown_content_metadata()),}}@@ -257,7 +264,7 @@ fn user_msg(text: &str) -> ResponseItem {text: text.to_string(),}],phase: None,- internal_chat_message_metadata_passthrough: None,+ internal_chat_message_metadata_passthrough: Some(unknown_content_metadata()),}}@@ -269,7 +276,7 @@ fn user_input_text_msg(text: &str) -> ResponseItem {text: text.to_string(),}],phase: None,- internal_chat_message_metadata_passthrough: None,+ internal_chat_message_metadata_passthrough: Some(unknown_content_metadata()),}}@@ -281,7 +288,7 @@ fn developer_msg(text: &str) -> ResponseItem {text: text.to_string(),}],phase: None,- internal_chat_message_metadata_passthrough: None,+ internal_chat_message_metadata_passthrough: Some(unknown_content_metadata()),}}@@ -296,7 +303,10 @@ fn developer_msg_with_fragments(texts: &[&str]) -> ResponseItem {}).collect(),phase: None,- internal_chat_message_metadata_passthrough: None,+ internal_chat_message_metadata_passthrough: Some(InternalChatMessageMetadataPassthrough {+ content_item_kinds: Some(vec![ContentItemKind("unknown".to_string()); texts.len()]),+ ..Default::default()+ }),}}@@ -397,41 +407,7 @@ fn filters_non_api_messages() {let a = assistant_msg("hello");h.record_items([&u, &a], policy);- let items = raw_items(&h);- assert_eq!(- items,- vec![- ResponseItem::Reasoning {- id: None,- summary: vec![ReasoningItemReasoningSummary::SummaryText {- text: "summary".to_string(),- }],- content: Some(vec![ReasoningItemContent::ReasoningText {- text: "thinking...".to_string(),- }]),- encrypted_content: None,- internal_chat_message_metadata_passthrough: None,- },- ResponseItem::Message {- id: None,- role: "user".to_string(),- content: vec![ContentItem::OutputText {- text: "hi".to_string()- }],- phase: None,- internal_chat_message_metadata_passthrough: None,- },- ResponseItem::Message {- id: None,- role: "assistant".to_string(),- content: vec![ContentItem::OutputText {- text: "hello".to_string()- }],- phase: None,- internal_chat_message_metadata_passthrough: None,- }- ]- );+ assert_eq!(raw_items(&h), vec![reasoning, u, a]);}#[test]@@ -680,7 +656,17 @@ fn for_prompt_strips_media_when_model_does_not_support_it() {},],phase: None,- internal_chat_message_metadata_passthrough: None,+ internal_chat_message_metadata_passthrough: Some(+ InternalChatMessageMetadataPassthrough {+ content_item_kinds: Some(vec![+ ContentItemKind("user.text".to_string()),+ ContentItemKind("user.image".to_string()),+ ContentItemKind("user.audio".to_string()),+ ContentItemKind("user.text".to_string()),+ ]),+ ..Default::default()+ },+ ),},ResponseItem::FunctionCall {id: None,@@ -764,7 +750,17 @@ fn for_prompt_strips_media_when_model_does_not_support_it() {},],phase: None,- internal_chat_message_metadata_passthrough: None,+ internal_chat_message_metadata_passthrough: Some(+ InternalChatMessageMetadataPassthrough {+ content_item_kinds: Some(vec![+ ContentItemKind("user.text".to_string()),+ ContentItemKind("images.unsupported".to_string()),+ ContentItemKind("audio.unsupported".to_string()),+ ContentItemKind("user.text".to_string()),+ ]),+ ..Default::default()+ },+ ),},ResponseItem::FunctionCall {id: None,@@ -867,7 +863,7 @@ fn for_prompt_strips_media_when_model_does_not_support_it() {audio_url: "data:audio/wav;base64,YXVkaW8=".to_string(),}],phase: None,- internal_chat_message_metadata_passthrough: None,+ internal_chat_message_metadata_passthrough: Some(unknown_content_metadata()),};let with_audio = create_history_with_items(vec![audio_message.clone()]);assert_eq!(@@ -914,7 +910,7 @@ fn for_prompt_preserves_image_generation_calls_when_images_are_supported() {text: "hi".to_string(),}],phase: None,- internal_chat_message_metadata_passthrough: None,+ internal_chat_message_metadata_passthrough: Some(unknown_content_metadata()),}]);@@ -951,7 +947,7 @@ fn for_prompt_clears_image_generation_result_when_images_are_unsupported() {text: "generate a lobster".to_string(),}],phase: None,- internal_chat_message_metadata_passthrough: None,+ internal_chat_message_metadata_passthrough: Some(unknown_content_metadata()),},ResponseItem::ImageGenerationCall {id: Some(ResponseItemId::with_suffix("ig", "123")),diff --git a/codex-rs/core/src/context_manager/normalize.rs b/codex-rs/core/src/context_manager/normalize.rsindex 0e693ce09f..7fc45ba217 100644--- a/codex-rs/core/src/context_manager/normalize.rs+++ b/codex-rs/core/src/context_manager/normalize.rs@@ -1,3 +1,5 @@+use codex_context_fragments::set_annotated_content;+use codex_context_fragments::to_annotated_content;use codex_history::ResponseItemEnvelope;use codex_protocol::ResponseItemId;use codex_protocol::models::ContentItem;@@ -8,13 +10,11 @@ use codex_protocol::openai_models::InputModality;use std::collections::HashSet;use uuid::Uuid;+use crate::context::ContextualUserFragment;+use crate::context::UnsupportedMedia;use crate::util::error_or_panic;use tracing::info;-const IMAGE_CONTENT_OMITTED_PLACEHOLDER: &str =- "image content omitted because you do not support image input";-const AUDIO_CONTENT_OMITTED_PLACEHOLDER: &str =- "audio content omitted because you do not support audio input";// Changing this value would change model-visible IDs and invalidate prompt caches.const SYNTHETIC_OUTPUT_ID_NAMESPACE: Uuid = Uuid::from_u128(0x90d38d3e_6a5b_4d52_bfe2_2f1e634bfac4);@@ -338,19 +338,16 @@ pub(crate) fn strip_images_when_unsupported(for envelope in items.iter_mut() {match &mut envelope.item {- ResponseItem::Message { content, .. } => {- let mut normalized_content = Vec::with_capacity(content.len());- for content_item in content.iter() {- match content_item {- ContentItem::InputImage { .. } => {- normalized_content.push(ContentItem::InputText {- text: IMAGE_CONTENT_OMITTED_PLACEHOLDER.to_string(),- });- }- _ => normalized_content.push(content_item.clone()),+ ResponseItem::Message { .. } => {+ let Some(mut content) = to_annotated_content(&mut envelope.item) else {+ continue;+ };+ for content_item in &mut content {+ if matches!(content_item.content(), ContentItem::InputImage { .. }) {+ *content_item = UnsupportedMedia::IMAGE.render_fragment().into_parts().1;}}- *content = normalized_content;+ let _ = set_annotated_content(&mut envelope.item, content);}ResponseItem::FunctionCallOutput { output, .. }| ResponseItem::CustomToolCallOutput { output, .. } => {@@ -361,7 +358,7 @@ pub(crate) fn strip_images_when_unsupported(FunctionCallOutputContentItem::InputImage { .. } => {normalized_content_items.push(FunctionCallOutputContentItem::InputText {- text: IMAGE_CONTENT_OMITTED_PLACEHOLDER.to_string(),+ text: UnsupportedMedia::IMAGE.render(),},);}@@ -391,14 +388,16 @@ pub(crate) fn strip_audio_when_unsupported(for envelope in items.iter_mut() {match &mut envelope.item {- ResponseItem::Message { content, .. } => {- for content_item in content.iter_mut() {- if matches!(content_item, ContentItem::InputAudio { .. }) {- *content_item = ContentItem::InputText {- text: AUDIO_CONTENT_OMITTED_PLACEHOLDER.to_string(),- };+ ResponseItem::Message { .. } => {+ let Some(mut content) = to_annotated_content(&mut envelope.item) else {+ continue;+ };+ for content_item in &mut content {+ if matches!(content_item.content(), ContentItem::InputAudio { .. }) {+ *content_item = UnsupportedMedia::AUDIO.render_fragment().into_parts().1;}}+ let _ = set_annotated_content(&mut envelope.item, content);}ResponseItem::FunctionCallOutput { output, .. }| ResponseItem::CustomToolCallOutput { output, .. } => {@@ -409,7 +408,7 @@ pub(crate) fn strip_audio_when_unsupported(FunctionCallOutputContentItem::InputAudio { .. }) {*content_item = FunctionCallOutputContentItem::InputText {- text: AUDIO_CONTENT_OMITTED_PLACEHOLDER.to_string(),+ text: UnsupportedMedia::AUDIO.render(),};}}diff --git a/codex-rs/core/src/session/tests.rs b/codex-rs/core/src/session/tests.rsindex b977ccb9ef..1fa912cc1c 100644--- a/codex-rs/core/src/session/tests.rs+++ b/codex-rs/core/src/session/tests.rs@@ -11312,7 +11312,7 @@ async fn sample_rollout((rollout_items,- live_history.for_prompt(&reconstruction_turn.model_info.input_modalities),+ strip_metadata_from_items(&raw_history_items(&live_history)),)}diff --git a/codex-rs/core/tests/suite/model_switching.rs b/codex-rs/core/tests/suite/model_switching.rsindex 4bd09d11ae..e5232fe736 100644--- a/codex-rs/core/tests/suite/model_switching.rs+++ b/codex-rs/core/tests/suite/model_switching.rs@@ -992,6 +992,7 @@ async fn model_change_from_multimodal_to_text_strips_prior_media_content() -> Reassert_eq!(requests.len(), 2, "expected two model requests");let first_request = requests.first().expect("expected first request");+ assert!(first_request.has_content_kinds(&["user.image", "user.audio", "user.text"]));assert!(!first_request.message_input_image_urls("user").is_empty(),"first request should include the uploaded image"@@ -1002,6 +1003,11 @@ async fn model_change_from_multimodal_to_text_strips_prior_media_content() -> Re);let second_request = requests.last().expect("expected second request");+ assert!(second_request.has_content_kinds(&[+ "images.unsupported",+ "audio.unsupported",+ "user.text",+ ]));assert!(second_request.message_input_image_urls("user").is_empty(),"second request should strip unsupported image content"