Skip to content

armgit / codex

Public Imported from GitHub
0

Lightweight coding agent that runs in your terminal

Rust

Preserve annotations when omitting unsupported media (#40277)

pakrym-oaiaec653daa987
diff --git a/codex-rs/core/src/context/mod.rs b/codex-rs/core/src/context/mod.rs
index a2a68544e7..378fe0cb08 100644
--- a/codex-rs/core/src/context/mod.rs
+++ b/codex-rs/core/src/context/mod.rs
@@ -38,6 +38,7 @@ mod rollout_budget;
mod subagent_notification;
mod token_budget_context;
mod turn_aborted;
+mod unsupported_media;
mod user_instructions;
mod user_shell_command;
pub(crate) mod world_state;
@@ -98,6 +99,7 @@ pub(crate) use token_budget_context::TokenBudgetContext;
pub(crate) use token_budget_context::TokenBudgetRemainingContext;
pub(crate) use token_budget_context::TokenBudgetReminder;
pub(crate) use turn_aborted::TurnAborted;
+pub(crate) use unsupported_media::UnsupportedMedia;
pub(crate) use user_instructions::UserInstructions;
pub(crate) use user_shell_command::UserShellCommand;
pub(crate) use world_state::ManagedDeveloperInstructions;
diff --git a/codex-rs/core/src/context/unsupported_media.rs b/codex-rs/core/src/context/unsupported_media.rs
new file mode 100644
index 0000000000..8b225649c0
--- /dev/null
+++ b/codex-rs/core/src/context/unsupported_media.rs
@@ -0,0 +1,42 @@
+use super::ContextualUserFragment;
+use codex_protocol::models::ContentItemKind;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub(crate) struct UnsupportedMedia {
+ text: &'static str,
+ content_kind: &'static str,
+}
+
+impl UnsupportedMedia {
+ pub(crate) const IMAGE: Self = Self {
+ text: "image content omitted because you do not support image input",
+ content_kind: "images.unsupported",
+ };
+
+ pub(crate) const AUDIO: Self = Self {
+ text: "audio content omitted because you do not support audio input",
+ content_kind: "audio.unsupported",
+ };
+}
+
+impl ContextualUserFragment for UnsupportedMedia {
+ fn content_kind(&self) -> ContentItemKind {
+ ContentItemKind(self.content_kind.to_string())
+ }
+
+ fn role(&self) -> &'static str {
+ "user"
+ }
+
+ fn markers(&self) -> (&'static str, &'static str) {
+ Self::type_markers()
+ }
+
+ fn type_markers() -> (&'static str, &'static str) {
+ ("", "")
+ }
+
+ fn body(&self) -> String {
+ self.text.to_string()
+ }
+}
diff --git a/codex-rs/core/src/context_manager/history_tests.rs b/codex-rs/core/src/context_manager/history_tests.rs
index 5a0c6e2602..06fa8b4b08 100644
--- a/codex-rs/core/src/context_manager/history_tests.rs
+++ b/codex-rs/core/src/context_manager/history_tests.rs
@@ -46,6 +46,13 @@ const EXEC_FORMAT_MAX_BYTES: usize = 10_000;
const EXEC_FORMAT_MAX_TOKENS: usize = 2_500;
const TEST_WAV_SAMPLE_RATE: u32 = 8_000;
+fn unknown_content_metadata() -> InternalChatMessageMetadataPassthrough {
+ InternalChatMessageMetadataPassthrough {
+ content_item_kinds: Some(vec![ContentItemKind("unknown".to_string())]),
+ ..Default::default()
+ }
+}
+
fn pcm_wav_data_url(sample_count: u32) -> (String, usize) {
let padding = sample_count % 2;
let mut bytes = Vec::new();
@@ -78,7 +85,7 @@ fn assistant_msg(text: &str) -> ResponseItem {
text: text.to_string(),
}],
phase: None,
- internal_chat_message_metadata_passthrough: None,
+ internal_chat_message_metadata_passthrough: Some(unknown_content_metadata()),
}
}
@@ -97,7 +104,7 @@ fn inter_agent_assistant_msg(text: &str) -> ResponseItem {
text: serde_json::to_string(&communication).unwrap(),
}],
phase: None,
- internal_chat_message_metadata_passthrough: None,
+ internal_chat_message_metadata_passthrough: Some(unknown_content_metadata()),
}
}
@@ -257,7 +264,7 @@ fn user_msg(text: &str) -> ResponseItem {
text: text.to_string(),
}],
phase: None,
- internal_chat_message_metadata_passthrough: None,
+ internal_chat_message_metadata_passthrough: Some(unknown_content_metadata()),
}
}
@@ -269,7 +276,7 @@ fn user_input_text_msg(text: &str) -> ResponseItem {
text: text.to_string(),
}],
phase: None,
- internal_chat_message_metadata_passthrough: None,
+ internal_chat_message_metadata_passthrough: Some(unknown_content_metadata()),
}
}
@@ -281,7 +288,7 @@ fn developer_msg(text: &str) -> ResponseItem {
text: text.to_string(),
}],
phase: None,
- internal_chat_message_metadata_passthrough: None,
+ internal_chat_message_metadata_passthrough: Some(unknown_content_metadata()),
}
}
@@ -296,7 +303,10 @@ fn developer_msg_with_fragments(texts: &[&str]) -> ResponseItem {
})
.collect(),
phase: None,
- internal_chat_message_metadata_passthrough: None,
+ internal_chat_message_metadata_passthrough: Some(InternalChatMessageMetadataPassthrough {
+ content_item_kinds: Some(vec![ContentItemKind("unknown".to_string()); texts.len()]),
+ ..Default::default()
+ }),
}
}
@@ -397,41 +407,7 @@ fn filters_non_api_messages() {
let a = assistant_msg("hello");
h.record_items([&u, &a], policy);
- let items = raw_items(&h);
- assert_eq!(
- items,
- vec![
- ResponseItem::Reasoning {
- id: None,
- summary: vec![ReasoningItemReasoningSummary::SummaryText {
- text: "summary".to_string(),
- }],
- content: Some(vec![ReasoningItemContent::ReasoningText {
- text: "thinking...".to_string(),
- }]),
- encrypted_content: None,
- internal_chat_message_metadata_passthrough: None,
- },
- ResponseItem::Message {
- id: None,
- role: "user".to_string(),
- content: vec![ContentItem::OutputText {
- text: "hi".to_string()
- }],
- phase: None,
- internal_chat_message_metadata_passthrough: None,
- },
- ResponseItem::Message {
- id: None,
- role: "assistant".to_string(),
- content: vec![ContentItem::OutputText {
- text: "hello".to_string()
- }],
- phase: None,
- internal_chat_message_metadata_passthrough: None,
- }
- ]
- );
+ assert_eq!(raw_items(&h), vec![reasoning, u, a]);
}
#[test]
@@ -680,7 +656,17 @@ fn for_prompt_strips_media_when_model_does_not_support_it() {
},
],
phase: None,
- internal_chat_message_metadata_passthrough: None,
+ internal_chat_message_metadata_passthrough: Some(
+ InternalChatMessageMetadataPassthrough {
+ content_item_kinds: Some(vec![
+ ContentItemKind("user.text".to_string()),
+ ContentItemKind("user.image".to_string()),
+ ContentItemKind("user.audio".to_string()),
+ ContentItemKind("user.text".to_string()),
+ ]),
+ ..Default::default()
+ },
+ ),
},
ResponseItem::FunctionCall {
id: None,
@@ -764,7 +750,17 @@ fn for_prompt_strips_media_when_model_does_not_support_it() {
},
],
phase: None,
- internal_chat_message_metadata_passthrough: None,
+ internal_chat_message_metadata_passthrough: Some(
+ InternalChatMessageMetadataPassthrough {
+ content_item_kinds: Some(vec![
+ ContentItemKind("user.text".to_string()),
+ ContentItemKind("images.unsupported".to_string()),
+ ContentItemKind("audio.unsupported".to_string()),
+ ContentItemKind("user.text".to_string()),
+ ]),
+ ..Default::default()
+ },
+ ),
},
ResponseItem::FunctionCall {
id: None,
@@ -867,7 +863,7 @@ fn for_prompt_strips_media_when_model_does_not_support_it() {
audio_url: "data:audio/wav;base64,YXVkaW8=".to_string(),
}],
phase: None,
- internal_chat_message_metadata_passthrough: None,
+ internal_chat_message_metadata_passthrough: Some(unknown_content_metadata()),
};
let with_audio = create_history_with_items(vec![audio_message.clone()]);
assert_eq!(
@@ -914,7 +910,7 @@ fn for_prompt_preserves_image_generation_calls_when_images_are_supported() {
text: "hi".to_string(),
}],
phase: None,
- internal_chat_message_metadata_passthrough: None,
+ internal_chat_message_metadata_passthrough: Some(unknown_content_metadata()),
}
]
);
@@ -951,7 +947,7 @@ fn for_prompt_clears_image_generation_result_when_images_are_unsupported() {
text: "generate a lobster".to_string(),
}],
phase: None,
- internal_chat_message_metadata_passthrough: None,
+ internal_chat_message_metadata_passthrough: Some(unknown_content_metadata()),
},
ResponseItem::ImageGenerationCall {
id: Some(ResponseItemId::with_suffix("ig", "123")),
diff --git a/codex-rs/core/src/context_manager/normalize.rs b/codex-rs/core/src/context_manager/normalize.rs
index 0e693ce09f..7fc45ba217 100644
--- a/codex-rs/core/src/context_manager/normalize.rs
+++ b/codex-rs/core/src/context_manager/normalize.rs
@@ -1,3 +1,5 @@
+use codex_context_fragments::set_annotated_content;
+use codex_context_fragments::to_annotated_content;
use codex_history::ResponseItemEnvelope;
use codex_protocol::ResponseItemId;
use codex_protocol::models::ContentItem;
@@ -8,13 +10,11 @@ use codex_protocol::openai_models::InputModality;
use std::collections::HashSet;
use uuid::Uuid;
+use crate::context::ContextualUserFragment;
+use crate::context::UnsupportedMedia;
use crate::util::error_or_panic;
use tracing::info;
-const IMAGE_CONTENT_OMITTED_PLACEHOLDER: &str =
- "image content omitted because you do not support image input";
-const AUDIO_CONTENT_OMITTED_PLACEHOLDER: &str =
- "audio content omitted because you do not support audio input";
// Changing this value would change model-visible IDs and invalidate prompt caches.
const SYNTHETIC_OUTPUT_ID_NAMESPACE: Uuid = Uuid::from_u128(0x90d38d3e_6a5b_4d52_bfe2_2f1e634bfac4);
@@ -338,19 +338,16 @@ pub(crate) fn strip_images_when_unsupported(
for envelope in items.iter_mut() {
match &mut envelope.item {
- ResponseItem::Message { content, .. } => {
- let mut normalized_content = Vec::with_capacity(content.len());
- for content_item in content.iter() {
- match content_item {
- ContentItem::InputImage { .. } => {
- normalized_content.push(ContentItem::InputText {
- text: IMAGE_CONTENT_OMITTED_PLACEHOLDER.to_string(),
- });
- }
- _ => normalized_content.push(content_item.clone()),
+ ResponseItem::Message { .. } => {
+ let Some(mut content) = to_annotated_content(&mut envelope.item) else {
+ continue;
+ };
+ for content_item in &mut content {
+ if matches!(content_item.content(), ContentItem::InputImage { .. }) {
+ *content_item = UnsupportedMedia::IMAGE.render_fragment().into_parts().1;
}
}
- *content = normalized_content;
+ let _ = set_annotated_content(&mut envelope.item, content);
}
ResponseItem::FunctionCallOutput { output, .. }
| ResponseItem::CustomToolCallOutput { output, .. } => {
@@ -361,7 +358,7 @@ pub(crate) fn strip_images_when_unsupported(
FunctionCallOutputContentItem::InputImage { .. } => {
normalized_content_items.push(
FunctionCallOutputContentItem::InputText {
- text: IMAGE_CONTENT_OMITTED_PLACEHOLDER.to_string(),
+ text: UnsupportedMedia::IMAGE.render(),
},
);
}
@@ -391,14 +388,16 @@ pub(crate) fn strip_audio_when_unsupported(
for envelope in items.iter_mut() {
match &mut envelope.item {
- ResponseItem::Message { content, .. } => {
- for content_item in content.iter_mut() {
- if matches!(content_item, ContentItem::InputAudio { .. }) {
- *content_item = ContentItem::InputText {
- text: AUDIO_CONTENT_OMITTED_PLACEHOLDER.to_string(),
- };
+ ResponseItem::Message { .. } => {
+ let Some(mut content) = to_annotated_content(&mut envelope.item) else {
+ continue;
+ };
+ for content_item in &mut content {
+ if matches!(content_item.content(), ContentItem::InputAudio { .. }) {
+ *content_item = UnsupportedMedia::AUDIO.render_fragment().into_parts().1;
}
}
+ let _ = set_annotated_content(&mut envelope.item, content);
}
ResponseItem::FunctionCallOutput { output, .. }
| ResponseItem::CustomToolCallOutput { output, .. } => {
@@ -409,7 +408,7 @@ pub(crate) fn strip_audio_when_unsupported(
FunctionCallOutputContentItem::InputAudio { .. }
) {
*content_item = FunctionCallOutputContentItem::InputText {
- text: AUDIO_CONTENT_OMITTED_PLACEHOLDER.to_string(),
+ text: UnsupportedMedia::AUDIO.render(),
};
}
}
diff --git a/codex-rs/core/src/session/tests.rs b/codex-rs/core/src/session/tests.rs
index b977ccb9ef..1fa912cc1c 100644
--- a/codex-rs/core/src/session/tests.rs
+++ b/codex-rs/core/src/session/tests.rs
@@ -11312,7 +11312,7 @@ async fn sample_rollout(
(
rollout_items,
- live_history.for_prompt(&reconstruction_turn.model_info.input_modalities),
+ strip_metadata_from_items(&raw_history_items(&live_history)),
)
}
diff --git a/codex-rs/core/tests/suite/model_switching.rs b/codex-rs/core/tests/suite/model_switching.rs
index 4bd09d11ae..e5232fe736 100644
--- a/codex-rs/core/tests/suite/model_switching.rs
+++ b/codex-rs/core/tests/suite/model_switching.rs
@@ -992,6 +992,7 @@ async fn model_change_from_multimodal_to_text_strips_prior_media_content() -> Re
assert_eq!(requests.len(), 2, "expected two model requests");
let first_request = requests.first().expect("expected first request");
+ assert!(first_request.has_content_kinds(&["user.image", "user.audio", "user.text"]));
assert!(
!first_request.message_input_image_urls("user").is_empty(),
"first request should include the uploaded image"
@@ -1002,6 +1003,11 @@ async fn model_change_from_multimodal_to_text_strips_prior_media_content() -> Re
);
let second_request = requests.last().expect("expected second request");
+ assert!(second_request.has_content_kinds(&[
+ "images.unsupported",
+ "audio.unsupported",
+ "user.text",
+ ]));
assert!(
second_request.message_input_image_urls("user").is_empty(),
"second request should strip unsupported image content"