Skip to content

armgit / codex

Public Imported from GitHub
0

Lightweight coding agent that runs in your terminal

Rust

Route prepared images through the attachment store (#46065)

Krish Chainani5e636ea760d1
diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock
index 4cf1efabc7..77470f0ae1 100644
--- a/codex-rs/Cargo.lock
+++ b/codex-rs/Cargo.lock
@@ -2489,6 +2489,7 @@ dependencies = [
"pretty_assertions",
"serde",
"tokio",
+ "tracing",
]
[[package]]
diff --git a/codex-rs/attachment-store/Cargo.toml b/codex-rs/attachment-store/Cargo.toml
index f2a1f4e5b9..dd66412fa1 100644
--- a/codex-rs/attachment-store/Cargo.toml
+++ b/codex-rs/attachment-store/Cargo.toml
@@ -14,6 +14,7 @@ workspace = true
[dependencies]
serde = { workspace = true, features = ["derive"] }
+tracing = { workspace = true }
[dev-dependencies]
pretty_assertions = { workspace = true }
diff --git a/codex-rs/attachment-store/src/lib.rs b/codex-rs/attachment-store/src/lib.rs
index df586a9fd1..c39374e829 100644
--- a/codex-rs/attachment-store/src/lib.rs
+++ b/codex-rs/attachment-store/src/lib.rs
@@ -158,6 +158,7 @@ impl fmt::Debug for UploadResult {
pub struct InlineAttachmentStore;
impl AttachmentStore for InlineAttachmentStore {
+ #[tracing::instrument(level = "trace", skip_all)]
fn upload(&self, request: UploadRequest) -> UploadFuture<'_> {
Box::pin(async move {
Ok(UploadResult::Inline {
@@ -166,6 +167,7 @@ impl AttachmentStore for InlineAttachmentStore {
})
}
+ #[tracing::instrument(level = "trace", skip_all)]
fn resolve<'a>(&'a self, request: ResolveRequest<'a>) -> ResolveFuture<'a> {
let file_id = request.file_id;
Box::pin(async move {
diff --git a/codex-rs/core/src/guardian/review_session.rs b/codex-rs/core/src/guardian/review_session.rs
index fa7c59eb66..7179aa3175 100644
--- a/codex-rs/core/src/guardian/review_session.rs
+++ b/codex-rs/core/src/guardian/review_session.rs
@@ -12,6 +12,7 @@ pub(crate) use setup::run_guardian_review_session;
mod context_policy;
use context_policy::ReviewContextPolicy;
+use std::borrow::Cow;
use std::collections::HashMap;
use std::collections::HashSet;
use std::path::PathBuf;
@@ -68,8 +69,7 @@ use crate::context::GuardianNodeReplPolicy;
use crate::context_manager::ContextManager;
use crate::environment_selection::TurnEnvironmentSnapshot;
use crate::image_preparation::ImagePreparationMode;
-use crate::image_preparation::ImageResizeNoticeMode;
-use crate::image_preparation::prepare_response_items;
+use crate::image_preparation::resize_image;
use crate::image_preparation::unified_image_budget_enabled;
use crate::session::SessionIo;
use crate::session::session::Session;
@@ -80,7 +80,6 @@ use codex_model_provider_info::ModelProviderInfo;
use codex_protocol::turn_input::TurnInputMode;
use codex_protocol::turn_input::TurnInputRequest;
use codex_protocol::turn_input::TurnInputSubmission;
-use codex_protocol::user_input::UserInput;
use codex_thread_store::PersistContext;
use codex_tools::normalize_output_image_detail;
use codex_utils_path_uri::PathUri;
@@ -486,7 +485,8 @@ async fn run_review_on_session(
let parent_history = params.parent_history.conversation_history_snapshot();
let history = if GuardianContextMode::from_history(parent_history.as_ref())
- == GuardianContextMode::ThreadOwned {
+ == GuardianContextMode::ThreadOwned
+ {
parent_history
} else {
params.parent_session.conversation_history_snapshot().await
@@ -515,7 +515,10 @@ async fn run_review_on_session(
_ => &[],
})
.filter_map(|item| match item {
- ContentItem::InputImage { image: ImageReference::Inline { image_url }, .. } => Some(image_url.as_str()),
+ ContentItem::InputImage {
+ image: ImageReference::Inline { image_url },
+ ..
+ } => Some(image_url.as_str()),
_ => None,
})
.collect::<HashSet<_>>();
@@ -547,27 +550,15 @@ async fn run_review_on_session(
Some(ImageDetail::Low) => Some(ImageDetail::High),
detail => detail,
};
- let mut prepared = vec![
- ResponseInputItem::from(vec![UserInput::Image {
- image: ImageReference::Inline {
- image_url: image_url.to_owned(),
- },
- detail: *detail,
- }])
- .into(),
- ];
- prepare_response_items(
- &mut prepared,
- mode,
- ImageResizeNoticeMode::Disabled,
- );
- let Some(ResponseItem::Message { content, .. }) = prepared.first() else {
- return false;
+ let prepared_image_url = match resize_image(image_url, detail, mode) {
+ Ok(Some(prepared)) => Cow::Owned(prepared.into_data_url()),
+ Ok(None) => Cow::Borrowed(image_url),
+ Err(error) => {
+ warn!(%error, "failed to prepare guardian review image");
+ return false;
+ }
};
- content.iter().any(|item| {
- matches!(item, ContentItem::InputImage { image: ImageReference::Inline { image_url }, .. }
- if !reviewer_image_urls.contains(image_url.as_str()))
- })
+ !reviewer_image_urls.contains(prepared_image_url.as_ref())
});
let prompt: ResponseItem =
ResponseInputItem::from(prompt_items.context.clone().into_user_inputs()?)
diff --git a/codex-rs/core/src/image_preparation.rs b/codex-rs/core/src/image_preparation.rs
index 97098705cb..ce96d19777 100644
--- a/codex-rs/core/src/image_preparation.rs
+++ b/codex-rs/core/src/image_preparation.rs
@@ -6,6 +6,9 @@ use crate::context::ResizedImage;
use crate::original_image_detail::can_request_original_image_detail;
use codex_analytics::ImageDetailSetting;
use codex_analytics::ImagePreparationMetadata;
+use codex_attachment_store::AttachmentStore;
+use codex_attachment_store::UploadRequest;
+use codex_attachment_store::UploadResult;
use codex_context_fragments::AnnotatedContent;
use codex_context_fragments::set_annotated_content;
use codex_context_fragments::to_annotated_content;
@@ -18,8 +21,10 @@ use codex_protocol::models::ImageDetail;
use codex_protocol::models::ImageReference;
use codex_protocol::models::ResponseItem;
use codex_protocol::openai_models::ModelInfo;
+use codex_utils_image::EncodedImage;
use codex_utils_image::ImageProcessingError;
use codex_utils_image::PromptImageMode;
+use codex_utils_image::data_url_from_bytes;
use codex_utils_image::load_data_url_for_prompt;
use tracing::warn;
@@ -65,8 +70,32 @@ struct PreparedImageResize {
prepared_height: u32,
}
+pub(crate) struct PreparedInlineImage {
+ encoded: EncodedImage,
+ effective_detail: ImageDetailSetting,
+ resize: Option<PreparedImageResize>,
+}
+
+impl PreparedInlineImage {
+ pub(crate) fn into_data_url(self) -> String {
+ self.encoded.into_data_url()
+ }
+
+ fn metadata(&self, origin: ImageOrigin<'_>) -> ImagePreparationMetadata {
+ ImagePreparationMetadata {
+ message_role: origin.message_role.map(str::to_string),
+ item_id: origin.item_id.map(str::to_string),
+ effective_detail: self.effective_detail,
+ source_width: self.encoded.source_width,
+ source_height: self.encoded.source_height,
+ prepared_width: self.encoded.width,
+ prepared_height: self.encoded.height,
+ }
+ }
+}
+
#[derive(Debug, thiserror::Error)]
-enum ImagePreparationError {
+pub(crate) enum ImagePreparationError {
#[error("remote image URLs are not supported")]
RemoteUrlUnsupported,
#[error("image detail `low` is not supported")]
@@ -88,33 +117,14 @@ impl ImagePreparationError {
}
}
-pub(crate) fn prepare_response_items(
+pub(crate) async fn prepare_response_items(
items: &mut Vec<ResponseItem>,
mode: ImagePreparationMode,
resize_notice_mode: ImageResizeNoticeMode,
+ image_store: &dyn AttachmentStore,
) -> Vec<ImagePreparationMetadata> {
let mut metadata = Vec::new();
let mut prepared_items = Vec::with_capacity(items.len());
- let prepare_tool_output =
- |output: &mut FunctionCallOutputPayload,
- item_id: Option<&str>,
- metadata: &mut Vec<ImagePreparationMetadata>| {
- output.content_items_mut().and_then(|content| {
- let resized_images = prepare_tool_output_content(
- content,
- ImageOrigin {
- message_role: None,
- item_id,
- },
- resize_notice_mode,
- metadata,
- mode,
- );
- (!resized_images.is_empty()).then(|| {
- ImageResizeNotice::new(ImageResizeNoticeSource::ToolOutput, resized_images)
- })
- })
- };
for mut item in std::mem::take(items) {
let mut annotated_content = to_annotated_content(&mut item);
let resize_notice = match &mut item {
@@ -135,7 +145,9 @@ pub(crate) fn prepare_response_items(
},
&mut metadata,
mode,
- );
+ image_store,
+ )
+ .await;
let _ = set_annotated_content(&mut item, content);
(!resized_images.is_empty()).then(|| {
ImageResizeNotice::new(ImageResizeNoticeSource::UserMessage, resized_images)
@@ -143,10 +155,30 @@ pub(crate) fn prepare_response_items(
}
ResponseItem::FunctionCallOutput {
call_id, output, ..
- } => prepare_tool_output(output, call_id.as_deref(), &mut metadata),
+ } => {
+ prepare_tool_output(
+ output,
+ call_id.as_deref(),
+ resize_notice_mode,
+ &mut metadata,
+ mode,
+ image_store,
+ )
+ .await
+ }
ResponseItem::CustomToolCallOutput {
call_id, output, ..
- } => prepare_tool_output(output, Some(call_id.as_str()), &mut metadata),
+ } => {
+ prepare_tool_output(
+ output,
+ Some(call_id.as_str()),
+ resize_notice_mode,
+ &mut metadata,
+ mode,
+ image_store,
+ )
+ .await
+ }
ResponseItem::AdditionalTools { .. }
| ResponseItem::Reasoning { .. }
| ResponseItem::AgentMessage { .. }
@@ -172,15 +204,47 @@ pub(crate) fn prepare_response_items(
metadata
}
-fn prepare_message_content(
+async fn prepare_tool_output(
+ output: &mut FunctionCallOutputPayload,
+ item_id: Option<&str>,
+ resize_notice_mode: ImageResizeNoticeMode,
+ metadata: &mut Vec<ImagePreparationMetadata>,
+ mode: ImagePreparationMode,
+ image_store: &dyn AttachmentStore,
+) -> Option<ImageResizeNotice> {
+ let content = output.content_items_mut()?;
+ let resized_images = prepare_tool_output_content(
+ content,
+ ImageOrigin {
+ message_role: None,
+ item_id,
+ },
+ resize_notice_mode,
+ metadata,
+ mode,
+ image_store,
+ )
+ .await;
+ if resized_images.is_empty() {
+ None
+ } else {
+ Some(ImageResizeNotice::new(
+ ImageResizeNoticeSource::ToolOutput,
+ resized_images,
+ ))
+ }
+}
+
+async fn prepare_message_content(
items: &mut [AnnotatedContent],
origin: ImageOrigin<'_>,
resize_notice_mode: ImageResizeNoticeMode,
metadata: &mut Vec<ImagePreparationMetadata>,
mode: ImagePreparationMode,
+ image_store: &dyn AttachmentStore,
) -> Vec<ResizedImage> {
- // TODO(kc) Prepare file-backed images once Core has an explicit resolution contract. Until
- // then they pass through while retaining their positions in resize notices.
+ // File-backed images are already prepared and pass through while retaining their positions in
+ // resize notices.
let image_count = items
.iter()
.filter(|item| matches!(item.content(), ContentItem::InputImage { .. }))
@@ -188,15 +252,9 @@ fn prepare_message_content(
let mut image_number = 0;
let mut resized_images = Vec::new();
for item in items {
- if matches!(item.content(), ContentItem::InputImage { .. }) {
+ if let ContentItem::InputImage { image, detail } = item.content_mut() {
image_number += 1;
- }
- if let ContentItem::InputImage {
- image: ImageReference::Inline { image_url },
- detail,
- } = item.content_mut()
- {
- match prepare_image(image_url, detail, origin, metadata, mode) {
+ match prepare_image(image, detail, origin, metadata, mode, image_store).await {
Ok(Some(resize)) if resize_notice_mode == ImageResizeNoticeMode::Enabled => {
resized_images.push(ResizedImage {
image_number,
@@ -221,12 +279,13 @@ fn prepare_message_content(
resized_images
}
-fn prepare_tool_output_content(
+async fn prepare_tool_output_content(
items: &mut [FunctionCallOutputContentItem],
origin: ImageOrigin<'_>,
resize_notice_mode: ImageResizeNoticeMode,
metadata: &mut Vec<ImagePreparationMetadata>,
mode: ImagePreparationMode,
+ image_store: &dyn AttachmentStore,
) -> Vec<ResizedImage> {
let image_count = items
.iter()
@@ -235,15 +294,9 @@ fn prepare_tool_output_content(
let mut image_number = 0;
let mut resized_images = Vec::new();
for item in items {
- if matches!(item, FunctionCallOutputContentItem::InputImage { .. }) {
+ if let FunctionCallOutputContentItem::InputImage { image, detail } = item {
image_number += 1;
- }
- if let FunctionCallOutputContentItem::InputImage {
- image: ImageReference::Inline { image_url },
- detail,
- } = item
- {
- match prepare_image(image_url, detail, origin, metadata, mode) {
+ match prepare_image(image, detail, origin, metadata, mode, image_store).await {
Ok(Some(resize)) if resize_notice_mode == ImageResizeNoticeMode::Enabled => {
resized_images.push(ResizedImage {
image_number,
@@ -279,13 +332,46 @@ fn is_data_url(image_url: &str) -> bool {
.is_some_and(|prefix| prefix.eq_ignore_ascii_case("data:"))
}
-fn prepare_image(
- image_url: &mut String,
+async fn prepare_image(
+ image: &mut ImageReference,
detail: &mut Option<ImageDetail>,
origin: ImageOrigin<'_>,
metadata: &mut Vec<ImagePreparationMetadata>,
mode: ImagePreparationMode,
+ image_store: &dyn AttachmentStore,
) -> Result<Option<PreparedImageResize>, ImagePreparationError> {
+ let ImageReference::Inline { image_url } = image else {
+ return Ok(None);
+ };
+ let Some(prepared) = resize_image(image_url, detail, mode)? else {
+ return Ok(None);
+ };
+ metadata.push(prepared.metadata(origin));
+ let resize = prepared.resize;
+ let request = UploadRequest {
+ file_name: None,
+ data: prepared.encoded.bytes.to_vec(),
+ };
+ *image = match image_store.upload(request).await {
+ Ok(UploadResult::Inline { bytes }) => ImageReference::Inline {
+ image_url: data_url_from_bytes(&prepared.encoded.mime, &bytes),
+ },
+ Ok(UploadResult::File { file_id }) => ImageReference::File { file_id },
+ Err(error) => {
+ warn!(kind = ?error.kind(), "failed to upload prepared image");
+ ImageReference::Inline {
+ image_url: prepared.into_data_url(),
+ }
+ }
+ };
+ Ok(resize)
+}
+
+pub(crate) fn resize_image(
+ image_url: &str,
+ detail: &mut Option<ImageDetail>,
+ mode: ImagePreparationMode,
+) -> Result<Option<PreparedInlineImage>, ImagePreparationError> {
if is_remote_image_url(image_url) {
return Err(ImagePreparationError::RemoteUrlUnsupported);
}
@@ -309,30 +395,24 @@ fn prepare_image(
Some(ImageDetail::Low) => return Err(ImagePreparationError::UnsupportedLowDetail),
},
};
- let image = load_data_url_for_prompt(image_url, image_mode)?;
- metadata.push(ImagePreparationMetadata {
- message_role: origin.message_role.map(str::to_string),
- item_id: origin.item_id.map(str::to_string),
- effective_detail,
- source_width: image.source_width,
- source_height: image.source_height,
- prepared_width: image.width,
- prepared_height: image.height,
- });
- let resize = ((image.source_width, image.source_height) != (image.width, image.height))
+ let encoded = load_data_url_for_prompt(image_url, image_mode)?;
+ let resize = ((encoded.source_width, encoded.source_height) != (encoded.width, encoded.height))
.then_some(PreparedImageResize {
- source_width: image.source_width,
- source_height: image.source_height,
- prepared_width: image.width,
- prepared_height: image.height,
+ source_width: encoded.source_width,
+ source_height: encoded.source_height,
+ prepared_width: encoded.width,
+ prepared_height: encoded.height,
});
- *image_url = image.into_data_url();
if mode == ImagePreparationMode::UnifiedBudget {
// Preserve accurate context-window accounting while older transports still require an
// image detail field. Responses Lite removes this compatibility hint before sending.
*detail = Some(ImageDetail::Original);
}
- Ok(resize)
+ Ok(Some(PreparedInlineImage {
+ encoded,
+ effective_detail,
+ resize,
+ }))
}
#[cfg(test)]
diff --git a/codex-rs/core/src/image_preparation_tests.rs b/codex-rs/core/src/image_preparation_tests.rs
index ba2324358d..f083fc4034 100644
--- a/codex-rs/core/src/image_preparation_tests.rs
+++ b/codex-rs/core/src/image_preparation_tests.rs
@@ -2,6 +2,12 @@ use std::io::Cursor;
use base64::Engine;
use base64::engine::general_purpose::STANDARD as BASE64_STANDARD;
+use codex_attachment_store::AttachmentStoreError;
+use codex_attachment_store::AttachmentStoreErrorKind;
+use codex_attachment_store::InlineAttachmentStore;
+use codex_attachment_store::ResolveFuture;
+use codex_attachment_store::ResolveRequest;
+use codex_attachment_store::UploadFuture;
use codex_protocol::models::ContentItemKind;
use codex_protocol::models::FunctionCallOutputBody;
use codex_protocol::models::FunctionCallOutputPayload;
@@ -17,6 +23,29 @@ use pretty_assertions::assert_eq;
use super::*;
+struct FailingAttachmentStore;
+
+impl AttachmentStore for FailingAttachmentStore {
+ fn upload(&self, _request: UploadRequest) -> UploadFuture<'_> {
+ Box::pin(async {
+ Err(AttachmentStoreError::new(
+ AttachmentStoreErrorKind::Backend,
+ "upload failed",
+ ))
+ })
+ }
+
+ fn resolve<'a>(&'a self, request: ResolveRequest<'a>) -> ResolveFuture<'a> {
+ let file_id = request.file_id;
+ Box::pin(async move {
+ Err(AttachmentStoreError::new(
+ AttachmentStoreErrorKind::NotFound,
+ format!("attachment `{file_id}` was not found"),
+ ))
+ })
+ }
+}
+
fn png_data_url(width: u32, height: u32) -> (String, Vec<u8>) {
let image = ImageBuffer::from_pixel(width, height, Rgba([10u8, 20, 30, 255]));
let mut encoded = Cursor::new(Vec::new());
@@ -34,8 +63,8 @@ fn decoded_image(image_url: &str) -> (Vec<u8>, DynamicImage) {
(bytes, image)
}
-#[test]
-fn preparation_preserves_small_image_bytes_and_replaces_remote_urls() {
+#[tokio::test(flavor = "multi_thread")]
+async fn preparation_preserves_small_image_bytes_and_replaces_remote_urls() {
let (data_url, original_bytes) = png_data_url(/*width*/ 64, /*height*/ 32);
let mut items = vec![ResponseItem::Message {
id: None,
@@ -74,7 +103,9 @@ fn preparation_preserves_small_image_bytes_and_replaces_remote_urls() {
&mut items,
ImagePreparationMode::DetailBased,
ImageResizeNoticeMode::Disabled,
- );
+ &InlineAttachmentStore,
+ )
+ .await;
let ResponseItem::Message { content, .. } = &items[0] else {
panic!("expected message");
@@ -112,8 +143,8 @@ fn preparation_preserves_small_image_bytes_and_replaces_remote_urls() {
);
}
-#[test]
-fn detail_policies_apply_the_expected_budgets() {
+#[tokio::test(flavor = "multi_thread")]
+async fn detail_policies_apply_the_expected_budgets() {
for (detail, effective_detail, input_dimensions, expected_dimensions) in [
(
Some(ImageDetail::High),
@@ -157,7 +188,9 @@ fn detail_policies_apply_the_expected_budgets() {
&mut items,
ImagePreparationMode::DetailBased,
ImageResizeNoticeMode::Disabled,
- );
+ &InlineAttachmentStore,
+ )
+ .await;
let ResponseItem::Message { content, .. } = &items[0] else {
panic!("expected message");
@@ -187,8 +220,8 @@ fn detail_policies_apply_the_expected_budgets() {
}
}
-#[test]
-fn preparation_reports_tool_output_item_id() {
+#[tokio::test(flavor = "multi_thread")]
+async fn preparation_reports_tool_output_item_id() {
let call_id = "call-image";
let (image_url, _) = png_data_url(/*width*/ 64, /*height*/ 32);
let mut items = vec![ResponseItem::FunctionCallOutput {
@@ -208,7 +241,9 @@ fn preparation_reports_tool_output_item_id() {
&mut items,
ImagePreparationMode::DetailBased,
ImageResizeNoticeMode::Disabled,
- );
+ &InlineAttachmentStore,
+ )
+ .await;
assert_eq!(
metadata,
@@ -224,9 +259,46 @@ fn preparation_reports_tool_output_item_id() {
);
}
+#[tokio::test(flavor = "multi_thread")]
+async fn upload_failure_keeps_resized_image_inline() {
+ let (image_url, _) = png_data_url(/*width*/ 2048, /*height*/ 2048);
+ let mut items = vec![ResponseItem::Message {
+ id: None,
+ role: "user".to_string(),
+ content: vec![ContentItem::InputImage {
+ image: ImageReference::Inline { image_url },
+ detail: Some(ImageDetail::High),
+ }],
+ phase: None,
+ internal_chat_message_metadata_passthrough: None,
+ }];
+
+ prepare_response_items(
+ &mut items,
+ ImagePreparationMode::DetailBased,
+ ImageResizeNoticeMode::Disabled,
+ &FailingAttachmentStore,
+ )
+ .await;
+
+ let ResponseItem::Message { content, .. } = &items[0] else {
+ panic!("expected message");
+ };
+ let [
+ ContentItem::InputImage {
+ image: ImageReference::Inline { image_url },
+ ..
+ },
+ ] = content.as_slice()
+ else {
+ panic!("expected inline image");
+ };
+ assert_eq!(decoded_image(image_url).1.dimensions(), (1600, 1600));
+}
+
/// File-backed images retain their sequence positions even though preparation skips them.
-#[test]
-fn resize_notices_count_file_backed_images_and_skip_failed_images() {
+#[tokio::test(flavor = "multi_thread")]
+async fn resize_notices_count_file_backed_images_and_skip_failed_images() {
let (large_image_url, _) = png_data_url(/*width*/ 2048, /*height*/ 2048);
let (small_image_url, _) = png_data_url(/*width*/ 64, /*height*/ 32);
let mut items = vec![
@@ -305,7 +377,9 @@ fn resize_notices_count_file_backed_images_and_skip_failed_images() {
&mut items,
ImagePreparationMode::DetailBased,
ImageResizeNoticeMode::Enabled,
- );
+ &InlineAttachmentStore,
+ )
+ .await;
let expected_user_notice = concat!(
"<image_resize_notice>\n",
"Image 4 of 4 in the preceding user message was resized from 2048x2048 to 1600x1600 pixels.\n",
@@ -443,8 +517,8 @@ fn resize_notices_count_file_backed_images_and_skip_failed_images() {
);
}
-#[test]
-fn preparation_replaces_only_failed_tool_images_and_preserves_metadata() {
+#[tokio::test(flavor = "multi_thread")]
+async fn preparation_replaces_only_failed_tool_images_and_preserves_metadata() {
let (valid_image_url, _) = png_data_url(/*width*/ 64, /*height*/ 32);
let expected_valid_image_url = valid_image_url.clone();
let mut items = vec![ResponseItem::CustomToolCallOutput {
@@ -490,7 +564,9 @@ fn preparation_replaces_only_failed_tool_images_and_preserves_metadata() {
&mut items,
ImagePreparationMode::DetailBased,
ImageResizeNoticeMode::Disabled,
- );
+ &InlineAttachmentStore,
+ )
+ .await;
assert_eq!(
items,
diff --git a/codex-rs/core/src/session/inject.rs b/codex-rs/core/src/session/inject.rs
index 50b306a774..8264d23886 100644
--- a/codex-rs/core/src/session/inject.rs
+++ b/codex-rs/core/src/session/inject.rs
@@ -127,11 +127,13 @@ impl Session {
let mut annotated_items = Vec::with_capacity(items.len());
let mut image_preparations = Vec::new();
for envelope in items {
- let (prepared_items, prepared_images) = self.prepare_conversation_items_for_history(
- turn_context,
- model_info,
- std::slice::from_ref(&envelope.item),
- );
+ let (prepared_items, prepared_images) = self
+ .prepare_conversation_items_for_history(
+ turn_context,
+ model_info,
+ std::slice::from_ref(&envelope.item),
+ )
+ .await;
image_preparations.extend(prepared_images);
let mut metadata = envelope.metadata;
diff --git a/codex-rs/core/src/session/mod.rs b/codex-rs/core/src/session/mod.rs
index 8a1679bddc..c242c83cd4 100644
--- a/codex-rs/core/src/session/mod.rs
+++ b/codex-rs/core/src/session/mod.rs
@@ -64,6 +64,7 @@ use codex_analytics::SubAgentThreadStartedInput;
use codex_analytics::TurnCodexErrorFact;
use codex_async_utils::OrCancelExt;
use codex_attachment_store::AttachmentStore;
+use codex_attachment_store::InlineAttachmentStore;
use codex_connectors::connector_runtime_context_key;
use codex_context_fragments::RenderedFragment;
use codex_exec_server::Environment;
@@ -1706,11 +1707,16 @@ impl Session {
.into_iter()
.map(|envelope| (envelope.item, envelope.metadata))
.unzip();
- let _ = prepare_image_response_items(
+ // Replay must not upload or migrate recorded history. The inline store returns prepared
+ // inline bytes, while existing file references bypass preparation and remain unchanged.
+ // Bound replay future size now that image preparation can await storage.
+ let _ = Box::pin(prepare_image_response_items(
&mut prepared_history,
ImagePreparationMode::DetailBased,
ImageResizeNoticeMode::Disabled,
- );
+ &InlineAttachmentStore,
+ ))
+ .await;
prepare_audio_response_items(&mut prepared_history);
assert_eq!(
prepared_history.len(),
@@ -3476,7 +3482,7 @@ impl Session {
}
/// Prepares media using the originating model and preserves existing item identity.
- pub(crate) fn prepare_conversation_items_for_history<'a>(
+ pub(crate) async fn prepare_conversation_items_for_history<'a>(
&self,
turn_context: &TurnContext,
model_info: &ModelInfo,
@@ -3498,11 +3504,14 @@ impl Session {
} else {
ImageResizeNoticeMode::Disabled
};
- let image_preparations = prepare_image_response_items(
+ // Keep nested image-upload futures out of every caller's future frame.
+ let image_preparations = Box::pin(prepare_image_response_items(
&mut items,
image_preparation_mode,
image_resize_notice_mode,
- );
+ self.services.image_store.as_ref(),
+ ))
+ .await;
prepare_audio_response_items(&mut items);
// Most response items get their passthrough turn ID at the durable history boundary.
for item in &mut items {
@@ -3591,8 +3600,9 @@ impl Session {
model_info: &ModelInfo,
items: &[ResponseItem],
) {
- let (items, image_preparations) =
- self.prepare_conversation_items_for_history(turn_context, model_info, items);
+ let (items, image_preparations) = self
+ .prepare_conversation_items_for_history(turn_context, model_info, items)
+ .await;
let items = items
.into_owned()
.into_iter()
@@ -3911,11 +3921,13 @@ impl Session {
communication: InterAgentCommunication,
) {
let response_item = communication.to_model_input_item();
- let (items, _) = self.prepare_conversation_items_for_history(
- turn_context,
- model_info,
- std::slice::from_ref(&response_item),
- );
+ let (items, _) = self
+ .prepare_conversation_items_for_history(
+ turn_context,
+ model_info,
+ std::slice::from_ref(&response_item),
+ )
+ .await;
let items = items.as_ref();
let response_item = items[0].clone();
{
diff --git a/codex-rs/core/src/session/tests.rs b/codex-rs/core/src/session/tests.rs
index a0c8c17014..394ef4b064 100644
--- a/codex-rs/core/src/session/tests.rs
+++ b/codex-rs/core/src/session/tests.rs
@@ -351,11 +351,13 @@ async fn default_turn_context_assigns_missing_response_item_ids() {
let (session, turn_context) = make_session_and_context().await;
let response_item = user_message("hello");
- let (items, _) = session.prepare_conversation_items_for_history(
- &turn_context,
- turn_context.model_info(),
- std::slice::from_ref(&response_item),
- );
+ let (items, _) = session
+ .prepare_conversation_items_for_history(
+ &turn_context,
+ turn_context.model_info(),
+ std::slice::from_ref(&response_item),
+ )
+ .await;
assert!(
items[0]
diff --git a/codex-rs/core/tests/suite/image_rollout.rs b/codex-rs/core/tests/suite/image_rollout.rs
index e9145c1d5d..b30ad11aa9 100644
--- a/codex-rs/core/tests/suite/image_rollout.rs
+++ b/codex-rs/core/tests/suite/image_rollout.rs
@@ -1,6 +1,14 @@
use anyhow::Context;
use base64::Engine;
use base64::engine::general_purpose::STANDARD as BASE64_STANDARD;
+use codex_attachment_store::AttachmentStore;
+use codex_attachment_store::AttachmentStoreError;
+use codex_attachment_store::AttachmentStoreErrorKind;
+use codex_attachment_store::ResolveFuture;
+use codex_attachment_store::ResolveRequest;
+use codex_attachment_store::UploadFuture;
+use codex_attachment_store::UploadRequest;
+use codex_attachment_store::UploadResult;
use codex_core::TurnInputRequest;
use codex_features::Feature;
use codex_history::RolloutItem;
@@ -42,8 +50,38 @@ use pretty_assertions::assert_eq;
use serde_json::Value;
use std::fs;
use std::path::Path;
+use std::sync::Arc;
+use std::sync::Mutex;
use std::time::Duration;
+const UPLOADED_FILE_ID: &str = "file_uploaded_image";
+
+#[derive(Default)]
+struct RecordingFileAttachmentStore {
+ uploads: Mutex<Vec<UploadRequest>>,
+}
+
+impl AttachmentStore for RecordingFileAttachmentStore {
+ fn upload(&self, request: UploadRequest) -> UploadFuture<'_> {
+ self.uploads.lock().expect("uploads lock").push(request);
+ Box::pin(async {
+ Ok(UploadResult::File {
+ file_id: UPLOADED_FILE_ID.to_string(),
+ })
+ })
+ }
+
+ fn resolve<'a>(&'a self, request: ResolveRequest<'a>) -> ResolveFuture<'a> {
+ let file_id = request.file_id;
+ Box::pin(async move {
+ Err(AttachmentStoreError::new(
+ AttachmentStoreErrorKind::NotFound,
+ format!("attachment `{file_id}` was not found"),
+ ))
+ })
+ }
+}
+
fn find_user_message_with_image(text: &str) -> Option<ResponseItem> {
for line in text.lines() {
let trimmed = line.trim();
@@ -437,12 +475,15 @@ async fn resumed_history_only_emits_resize_notices_for_new_images() -> anyhow::R
.join("\n");
fs::write(&rollout_path, format!("{rollout}\n"))?;
- let mut resume_builder = test_codex().with_config(|config| {
- let _ = config.features.enable(Feature::ImageResizeNotice);
- let _ = config
- .features
- .enable(Feature::RetainClientDeveloperMessages);
- });
+ let resume_image_store = Arc::new(RecordingFileAttachmentStore::default());
+ let mut resume_builder = test_codex()
+ .with_image_store(resume_image_store.clone())
+ .with_config(|config| {
+ let _ = config.features.enable(Feature::ImageResizeNotice);
+ let _ = config
+ .features
+ .enable(Feature::RetainClientDeveloperMessages);
+ });
let resumed = resume_builder
.resume(&server, initial.home.clone(), rollout_path.clone())
.await?;
@@ -468,6 +509,16 @@ async fn resumed_history_only_emits_resize_notices_for_new_images() -> anyhow::R
matches!(event, EventMsg::TurnComplete(_))
})
.await;
+ {
+ let uploads = resume_image_store.uploads.lock().expect("uploads lock");
+ let [upload] = uploads.as_slice() else {
+ panic!("only the new image should be uploaded");
+ };
+ assert_eq!(
+ image::load_from_memory(&upload.data)?.dimensions(),
+ (2048, 768)
+ );
+ }
let request = resumed_mock.single_request();
assert!(request.has_content_kinds(&["images.resize_notice"]));
@@ -507,6 +558,15 @@ async fn resumed_history_only_emits_resize_notices_for_new_images() -> anyhow::R
.context("historical image data URL")?;
let historical_image = image::load_from_memory(&BASE64_STANDARD.decode(encoded_image)?)?;
assert_eq!(historical_image.dimensions(), (2048, 768));
+ assert_eq!(
+ input[image_message_indices[1]]
+ .get("content")
+ .and_then(Value::as_array)
+ .and_then(|content| content.first())
+ .and_then(|item| item.get("file_id"))
+ .and_then(Value::as_str),
+ Some(UPLOADED_FILE_ID)
+ );
let expected_notice = concat!(
"<image_resize_notice>\n",
@@ -543,8 +603,17 @@ async fn resumed_history_only_emits_resize_notices_for_new_images() -> anyhow::R
)
.await;
replayed.submit_turn("preserve recorded notices").await?;
- let replayed_notices = replayed_mock
- .single_request()
+ let replayed_request = replayed_mock.single_request();
+ assert!(replayed_request.input().iter().any(|item| {
+ item.get("content")
+ .and_then(Value::as_array)
+ .is_some_and(|content| {
+ content.iter().any(|item| {
+ item.get("file_id").and_then(Value::as_str) == Some(UPLOADED_FILE_ID)
+ })
+ })
+ }));
+ let replayed_notices = replayed_request
.message_input_texts("developer")
.into_iter()
.filter(|text| text.starts_with("<image_resize_notice>"))