Lightweight coding agent that runs in your terminal
Rust
Preserve uploaded image file IDs in user message display history (#46258)
Krish Chainani96aca987f72e
diff --git a/codex-rs/core/src/session/inject.rs b/codex-rs/core/src/session/inject.rsindex 8264d23886..d66602b205 100644--- a/codex-rs/core/src/session/inject.rs+++ b/codex-rs/core/src/session/inject.rs@@ -1,6 +1,7 @@use super::TurnInput as PendingTurnInput;use super::session::Session;use super::turn_context::TurnContext;+use codex_analytics::ImagePreparationMetadata;use codex_features::Feature;use codex_history::CodexHarnessMetadata;use codex_history::ResponseItemEnvelope;@@ -124,6 +125,24 @@ impl Session {return;}+ let (annotated_items, image_preparations) = self+ .prepare_annotated_conversation_items_for_history(turn_context, model_info, items)+ .await;+ self.record_prepared_conversation_items(+ turn_context,+ model_info,+ annotated_items,+ image_preparations,+ )+ .await;+ }++ pub(super) async fn prepare_annotated_conversation_items_for_history(+ &self,+ turn_context: &TurnContext,+ model_info: &ModelInfo,+ items: Vec<ResponseItemEnvelope>,+ ) -> (Vec<ResponseItemEnvelope>, Vec<ImagePreparationMetadata>) {let mut annotated_items = Vec::with_capacity(items.len());let mut image_preparations = Vec::new();for envelope in items {@@ -144,13 +163,7 @@ impl Session {}}));}- self.record_prepared_conversation_items(- turn_context,- model_info,- annotated_items,- image_preparations,- )- .await;+ (annotated_items, image_preparations)}/// Injects items into active work, or records them without starting a turn.diff --git a/codex-rs/core/src/session/mod.rs b/codex-rs/core/src/session/mod.rsindex 3b22567947..7a19a9078d 100644--- a/codex-rs/core/src/session/mod.rs+++ b/codex-rs/core/src/session/mod.rs@@ -122,6 +122,7 @@ use codex_protocol::models::BaseInstructions;use codex_protocol::models::BaseInstructionsProvenance;use codex_protocol::models::ContentItem;use codex_protocol::models::ContentItemKind;+use codex_protocol::models::ImageReference;use codex_protocol::models::InternalChatMessageMetadataPassthrough;use codex_protocol::models::PermissionProfile;use codex_protocol::models::SandboxEnforcement;@@ -3548,9 +3549,18 @@ impl Session {}pub(crate) fn response_item_from_user_input(&self, input: Vec<UserInput>) -> ResponseItem {+ self.response_item_from_user_input_with_image_positions(input, &mut HashMap::new())+ }++ fn response_item_from_user_input_with_image_positions(+ &self,+ input: Vec<UserInput>,+ user_image_content_indices: &mut HashMap<usize, usize>,+ ) -> ResponseItem {let mut item = ResponseItem::from(ResponseInputItem::from_user_input(input,LocalImagePreparation::Defer,+ user_image_content_indices,));if let ResponseItem::Message {content,@@ -4843,20 +4853,37 @@ impl Session {// Persist the user message to history, but emit the turn item from `UserInput` so// UI-only `text_elements` are preserved. `ResponseItem::Message` does not carry// those spans, and `record_response_item_and_emit_turn_item` would drop them.- let response_item = self.response_item_from_user_input(input.to_vec());- self.record_annotated_conversation_items(+ let mut user_image_content_indices = HashMap::new();+ let response_item = self.response_item_from_user_input_with_image_positions(+ input.to_vec(),+ &mut user_image_content_indices,+ );+ let (prepared_items, image_preparations) = self+ .prepare_annotated_conversation_items_for_history(+ turn_context,+ model_info,+ vec![ResponseItemEnvelope {+ item: response_item,+ metadata: acceptance_order.map(|order| CodexHarnessMetadata {+ user_input_order: Some(order),+ ..Default::default()+ }),+ }],+ )+ .await;+ let mut user_message_item = UserMessageItem::new(input);+ apply_prepared_image_file_ids(+ &mut user_message_item,+ &prepared_items,+ &user_image_content_indices,+ );+ self.record_prepared_conversation_items(turn_context,model_info,- vec![ResponseItemEnvelope {- item: response_item,- metadata: acceptance_order.map(|order| CodexHarnessMetadata {- user_input_order: Some(order),- ..Default::default()- }),- }],+ prepared_items,+ image_preparations,).await;- let mut user_message_item = UserMessageItem::new(input);user_message_item.client_id = client_id;let turn_item = TurnItem::UserMessage(user_message_item);self.emit_turn_item_started(turn_context, &turn_item).await;@@ -4943,6 +4970,37 @@ impl Session {}}+fn apply_prepared_image_file_ids(+ user_message_item: &mut UserMessageItem,+ prepared_items: &[ResponseItemEnvelope],+ user_image_content_indices: &HashMap<usize, usize>,+) {+ // Preparation replaces image slots in place; resize notices are separate messages.+ // Read the references from the original message while retaining UI-only input spans.+ if let Some(ResponseItemEnvelope {+ item: ResponseItem::Message { content, .. },+ ..+ }) = prepared_items.first()+ {+ for (&input_index, &content_index) in user_image_content_indices {+ if let Some(ContentItem::InputImage {+ image: ImageReference::File { file_id },+ ..+ }) = content.get(content_index)+ && let UserInput::Image { detail, .. } | UserInput::LocalImage { detail, .. } =+ &user_message_item.content[input_index]+ {+ user_message_item.content[input_index] = UserInput::Image {+ image: ImageReference::File {+ file_id: file_id.clone(),+ },+ detail: *detail,+ };+ }+ }+ }+}+pub(crate) fn emit_subagent_session_started(analytics_events_client: &AnalyticsEventsClient,client_metadata: AppServerClientMetadata,diff --git a/codex-rs/core/tests/suite/image_rollout.rs b/codex-rs/core/tests/suite/image_rollout.rsindex fb749dc813..806f04bc81 100644--- a/codex-rs/core/tests/suite/image_rollout.rs+++ b/codex-rs/core/tests/suite/image_rollout.rs@@ -15,6 +15,7 @@ use codex_history::RolloutItem;use codex_protocol::config_types::CollaborationMode;use codex_protocol::config_types::ModeKind;use codex_protocol::config_types::Settings;+use codex_protocol::items::TurnItem;use codex_protocol::models::ContentItem;use codex_protocol::models::DEFAULT_IMAGE_DETAIL;use codex_protocol::models::ImageDetail;@@ -25,7 +26,10 @@ use codex_protocol::models::ResponseItem;use codex_protocol::protocol::AskForApproval;use codex_protocol::protocol::EventMsg;use codex_protocol::protocol::Op;+use codex_protocol::protocol::ThreadHistoryMode;use codex_protocol::protocol::ThreadSettingsOverrides;+use codex_protocol::user_input::ByteRange;+use codex_protocol::user_input::TextElement;use codex_protocol::user_input::UserInput;use codex_utils_image::data_url_from_bytes;use core_test_support::TempDirExt;@@ -43,6 +47,7 @@ use core_test_support::test_codex::local_selections;use core_test_support::test_codex::test_codex;use core_test_support::test_codex::turn_permission_fields;use core_test_support::wait_for_event;+use core_test_support::wait_for_event_match;use image::GenericImageView;use image::ImageBuffer;use image::Rgba;@@ -406,6 +411,166 @@ async fn file_image_passes_through_request_and_rollout() -> anyhow::Result<()> {Ok(())}+/// Uploaded images keep their original input positions and text spans in live and durable display+/// history, even when earlier images fail preparation or expand into multiple model content items.+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]+async fn uploaded_images_preserve_user_message_display_history() -> anyhow::Result<()> {+ skip_if_no_network!(Ok(()));++ let server = start_mock_server().await;+ let test = test_codex()+ .with_image_store(Arc::new(RecordingFileAttachmentStore::default()))+ .with_history_mode(ThreadHistoryMode::Paginated)+ .build_with_auto_env(&server)+ .await?;+ let image_path = test.cwd.path().join("display-image.png");+ write_test_png(&image_path, [12, 34, 56, 255])?;+ let image_url = data_url_from_bytes("image/png", &fs::read(&image_path)?);+ let input = vec![+ UserInput::Text {+ text: "inspect these images".to_string(),+ text_elements: vec![TextElement::new(+ ByteRange { start: 0, end: 7 },+ Some("<file>".to_string()),+ )],+ },+ UserInput::LocalImage {+ path: test.cwd.path().join("missing-image.png"),+ detail: None,+ },+ UserInput::Image {+ image: ImageReference::Inline {+ image_url: "data:image/png;base64,bm90IGFuIGltYWdl".to_string(),+ },+ detail: None,+ },+ UserInput::LocalImage {+ path: image_path,+ detail: Some(ImageDetail::High),+ },+ UserInput::Image {+ image: ImageReference::Inline {+ image_url: image_url.clone(),+ },+ detail: None,+ },+ UserInput::Text {+ text: "and this copy".to_string(),+ text_elements: Vec::new(),+ },+ UserInput::Image {+ image: ImageReference::File {+ file_id: "file_existing".to_string(),+ },+ detail: Some(ImageDetail::Original),+ },+ UserInput::Image {+ image: ImageReference::Inline { image_url },+ detail: None,+ },+ ];+ let mut expected_input = input.clone();+ for index in [3, 4, 7] {+ let detail = match &input[index] {+ UserInput::Image { detail, .. } | UserInput::LocalImage { detail, .. } => *detail,+ _ => unreachable!("these inputs are images"),+ };+ expected_input[index] = UserInput::Image {+ image: ImageReference::File {+ file_id: UPLOADED_FILE_ID.to_string(),+ },+ detail,+ };+ }+ let response_mock = responses::mount_sse_once(+ &server,+ sse(vec![+ ev_response_created("resp-display"),+ ev_completed("resp-display"),+ ]),+ )+ .await;+ test.codex+ .start_or_steer_turn(TurnInputRequest::user_input(input))+ .await?;++ let started = wait_for_event_match(&test.codex, |event| match event {+ EventMsg::ItemStarted(event) => match &event.item {+ TurnItem::UserMessage(item) => Some(item.clone()),+ _ => None,+ },+ _ => None,+ })+ .await;+ let completed = wait_for_event_match(&test.codex, |event| match event {+ EventMsg::ItemCompleted(event) => match &event.item {+ TurnItem::UserMessage(item) => Some(item.clone()),+ _ => None,+ },+ _ => None,+ })+ .await;+ assert_eq!(started.content, expected_input);+ assert_eq!(completed.id, started.id);+ assert_eq!(completed.client_id, started.client_id);+ assert_eq!(completed.content, started.content);+ wait_for_event(&test.codex, |event| {+ matches!(event, EventMsg::TurnComplete(_))+ })+ .await;++ let request = response_mock.single_request();+ let mut model_file_ids = Vec::new();+ for item in request.input() {+ if item.get("role").and_then(Value::as_str) != Some("user") {+ continue;+ }+ if let Some(content) = item.get("content").and_then(Value::as_array) {+ for content_item in content {+ if content_item.get("type").and_then(Value::as_str) == Some("input_image") {+ model_file_ids.push(+ content_item+ .get("file_id")+ .and_then(Value::as_str)+ .context("prepared image file ID")?+ .to_string(),+ );+ }+ }+ }+ }+ assert_eq!(+ model_file_ids,+ vec![+ UPLOADED_FILE_ID,+ UPLOADED_FILE_ID,+ "file_existing",+ UPLOADED_FILE_ID,+ ]+ );++ test.codex.shutdown_and_wait().await?;+ let rollout_path = test.codex.rollout_path().context("rollout path")?;+ let rollout_text = read_rollout_text(&rollout_path).await?;+ let mut persisted_user_items = Vec::new();+ for line in rollout_text.lines() {+ if let RolloutItem::EventMsg(EventMsg::ItemCompleted(event)) =+ codex_rollout::parse_rollout_line(line)?.item+ && let TurnItem::UserMessage(item) = event.item+ {+ persisted_user_items.push(item);+ }+ }+ let [persisted] = persisted_user_items.as_slice() else {+ panic!("expected exactly one persisted user message");+ };+ assert_eq!(persisted.id, completed.id);+ assert_eq!(persisted.client_id, completed.client_id);+ assert_eq!(persisted.content, completed.content);++ Ok(())+}+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]async fn resumed_history_only_emits_resize_notices_for_new_images() -> anyhow::Result<()> {skip_if_no_network!(Ok(()));diff --git a/codex-rs/protocol/src/models.rs b/codex-rs/protocol/src/models.rsindex ee7fecbf06..d51ded354c 100644--- a/codex-rs/protocol/src/models.rs+++ b/codex-rs/protocol/src/models.rs@@ -1993,78 +1993,91 @@ pub enum ReasoningItemContent {impl From<Vec<UserInput>> for ResponseInputItem {fn from(items: Vec<UserInput>) -> Self {- Self::from_user_input(items, LocalImagePreparation::Process)+ Self::from_user_input(items, LocalImagePreparation::Process, &mut HashMap::new())}}impl ResponseInputItem {+ /// Records original user-input indices to image slots in the generated message content.+ /// Inputs that do not produce an image, including failed local reads, have no entry.pub fn from_user_input(items: Vec<UserInput>,local_image_preparation: LocalImagePreparation,+ user_image_content_indices: &mut HashMap<usize, usize>,) -> Self {+ user_image_content_indices.clear();let mut image_index = 0;let mut audio_index = 0;- Self::Message {- role: "user".to_string(),- content: items- .into_iter()- .flat_map(|c| match c {- UserInput::Text { text, .. } => vec![ContentItem::InputText { text }],- UserInput::Image { image, detail, .. } => {- image_index += 1;- let detail = detail.unwrap_or(DEFAULT_IMAGE_DETAIL);- vec![ContentItem::InputImage {- image,- detail: Some(detail),- }]- }- UserInput::LocalImage { path, detail, .. } => {- image_index += 1;- let detail = detail.unwrap_or(DEFAULT_IMAGE_DETAIL);- match std::fs::read(&path) {- Ok(file_bytes) => match local_image_preparation {- LocalImagePreparation::Process => {- local_image_content_items_with_label_number(- &path,- file_bytes,- Some(image_index),- detail,- )- }- LocalImagePreparation::Defer => local_image_content_items(+ let mut content = Vec::new();+ for (input_index, input) in items.into_iter().enumerate() {+ let generated = match input {+ UserInput::Text { text, .. } => vec![ContentItem::InputText { text }],+ UserInput::Image { image, detail, .. } => {+ image_index += 1;+ let detail = detail.unwrap_or(DEFAULT_IMAGE_DETAIL);+ vec![ContentItem::InputImage {+ image,+ detail: Some(detail),+ }]+ }+ UserInput::LocalImage { path, detail, .. } => {+ image_index += 1;+ let detail = detail.unwrap_or(DEFAULT_IMAGE_DETAIL);+ match std::fs::read(&path) {+ Ok(file_bytes) => match local_image_preparation {+ LocalImagePreparation::Process => {+ local_image_content_items_with_label_number(&path,- data_url_from_bytes("application/octet-stream", &file_bytes),+ file_bytes,Some(image_index),detail,- ),- },- Err(err) => vec![local_media_error_placeholder(- &path,- err,- LocalMediaKind::Image,- )],- }- }- UserInput::Audio { audio_url } => {- audio_index += 1;- vec![ContentItem::InputAudio { audio_url }]- }- UserInput::LocalAudio { path } => {- audio_index += 1;- match std::fs::read(&path) {- Ok(file_bytes) => {- local_audio_content_items(&path, &file_bytes, audio_index)+ )}- Err(err) => vec![local_media_error_placeholder(+ LocalImagePreparation::Defer => local_image_content_items(&path,- err,- LocalMediaKind::Audio,- )],+ data_url_from_bytes("application/octet-stream", &file_bytes),+ Some(image_index),+ detail,+ ),+ },+ Err(err) => vec![local_media_error_placeholder(+ &path,+ err,+ LocalMediaKind::Image,+ )],+ }+ }+ UserInput::Audio { audio_url } => {+ audio_index += 1;+ vec![ContentItem::InputAudio { audio_url }]+ }+ UserInput::LocalAudio { path } => {+ audio_index += 1;+ match std::fs::read(&path) {+ Ok(file_bytes) => {+ local_audio_content_items(&path, &file_bytes, audio_index)}+ Err(err) => vec![local_media_error_placeholder(+ &path,+ err,+ LocalMediaKind::Audio,+ )],}- UserInput::Skill { .. } | UserInput::Mention { .. } => Vec::new(), // Tool bodies are injected later in core- })- .collect::<Vec<ContentItem>>(),+ }+ UserInput::Skill { .. } | UserInput::Mention { .. } => Vec::new(), // Tool bodies are injected later in core+ };+ // Local images add framing text, while mentions and skills add no content.+ // Capture the image's actual slot before preparation replaces its reference.+ for (offset, item) in generated.iter().enumerate() {+ if matches!(item, ContentItem::InputImage { .. }) {+ user_image_content_indices.insert(input_index, content.len() + offset);+ }+ }+ content.extend(generated);+ }+ Self::Message {+ role: "user".to_string(),+ content,phase: None,}}@@ -4259,6 +4272,61 @@ mod tests {Ok(())}+ /// Image associations use original input and expanded content positions, so omitted inputs,+ /// local-image framing, failed reads, and duplicate URLs cannot shift a later substitution.+ #[test]+ fn user_input_image_indices_survive_content_expansion() -> Result<()> {+ let dir = tempdir()?;+ let local_path = dir.path().join("local.png");+ std::fs::write(&local_path, TINY_PNG_BYTES)?;+ let inline_image = UserInput::Image {+ image: ImageReference::Inline {+ image_url: data_url_from_bytes("image/png", TINY_PNG_BYTES),+ },+ detail: None,+ };+ let inputs = vec![+ UserInput::Text {+ text: "compare images".to_string(),+ text_elements: Vec::new(),+ },+ UserInput::Mention {+ name: "app".to_string(),+ path: "app://example".to_string(),+ },+ UserInput::Skill {+ name: "skill".to_string(),+ path: dir.path().join("SKILL.md"),+ },+ UserInput::LocalImage {+ path: dir.path().join("missing.png"),+ detail: None,+ },+ UserInput::LocalImage {+ path: local_path,+ detail: None,+ },+ inline_image.clone(),+ inline_image,+ ];+ for preparation in [LocalImagePreparation::Defer, LocalImagePreparation::Process] {+ let mut indices = HashMap::new();+ let item =+ ResponseInputItem::from_user_input(inputs.clone(), preparation, &mut indices);+ assert_eq!(indices, HashMap::from([(4, 3), (5, 5), (6, 6)]));+ let ResponseInputItem::Message { content, .. } = item else {+ panic!("expected a user message");+ };+ for content_index in indices.into_values() {+ assert!(matches!(+ content[content_index],+ ContentItem::InputImage { .. }+ ));+ }+ }+ Ok(())+ }+#[test]fn mixed_remote_and_local_images_share_label_sequence() -> Result<()> {let image_url = "data:image/png;base64,abc".to_string();