Skip to content

armgit / codex

Public Imported from GitHub
0

Lightweight coding agent that runs in your terminal

Rust

Support file references in image edit requests (#47956)

Krish Chainani51d66fdd3b75
diff --git a/codex-rs/codex-api/src/endpoint/images.rs b/codex-rs/codex-api/src/endpoint/images.rs
index ce52f4ccbe..2217ce8961 100644
--- a/codex-rs/codex-api/src/endpoint/images.rs
+++ b/codex-rs/codex-api/src/endpoint/images.rs
@@ -122,13 +122,13 @@ mod tests {
use crate::images::ImageBackground;
use crate::images::ImageData;
use crate::images::ImageQuality;
- use crate::images::ImageUrl;
use crate::provider::RetryConfig;
use codex_client::Request;
use codex_client::RequestBody;
use codex_client::Response;
use codex_client::StreamResponse;
use codex_client::TransportError;
+ use codex_protocol::models::ImageReference;
use http::StatusCode;
use pretty_assertions::assert_eq;
use serde_json::json;
@@ -339,9 +339,14 @@ mod tests {
let (response, imagegen_request_id) = client
.edit(
&ImageEditRequest {
- images: vec![ImageUrl {
- image_url: "data:image/png;base64,Zm9v".to_string(),
- }],
+ images: vec![
+ ImageReference::Inline {
+ image_url: "data:image/png;base64,Zm9v".to_string(),
+ },
+ ImageReference::File {
+ file_id: "file-image".to_string(),
+ },
+ ],
prompt: "add a red hat".to_string(),
background: None,
model: "gpt-image-1.5".to_string(),
@@ -362,7 +367,10 @@ mod tests {
assert_eq!(
request.body.as_ref().and_then(RequestBody::json),
Some(&json!({
- "images": [{"image_url": "data:image/png;base64,Zm9v"}],
+ "images": [
+ {"image_url": "data:image/png;base64,Zm9v"},
+ {"file_id": "file-image"},
+ ],
"prompt": "add a red hat",
"model": "gpt-image-1.5",
}))
diff --git a/codex-rs/codex-api/src/images.rs b/codex-rs/codex-api/src/images.rs
index 96c7c37543..3f14d17334 100644
--- a/codex-rs/codex-api/src/images.rs
+++ b/codex-rs/codex-api/src/images.rs
@@ -1,3 +1,4 @@
+use codex_protocol::models::ImageReference;
use serde::Deserialize;
use serde::Serialize;
@@ -17,7 +18,7 @@ pub struct ImageGenerationRequest {
#[derive(Debug, Clone, Serialize, PartialEq)]
pub struct ImageEditRequest {
- pub images: Vec<ImageUrl>,
+ pub images: Vec<ImageReference>,
pub prompt: String,
#[serde(skip_serializing_if = "Option::is_none")]
pub background: Option<ImageBackground>,
@@ -30,11 +31,6 @@ pub struct ImageEditRequest {
pub size: Option<String>,
}
-#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
-pub struct ImageUrl {
- pub image_url: String,
-}
-
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
#[serde(rename_all = "lowercase")]
pub enum ImageBackground {
diff --git a/codex-rs/codex-api/src/lib.rs b/codex-rs/codex-api/src/lib.rs
index 12b55ca1d2..05df1c2708 100644
--- a/codex-rs/codex-api/src/lib.rs
+++ b/codex-rs/codex-api/src/lib.rs
@@ -83,7 +83,6 @@ pub use crate::images::ImageEditRequest;
pub use crate::images::ImageGenerationRequest;
pub use crate::images::ImageQuality;
pub use crate::images::ImageResponse;
-pub use crate::images::ImageUrl;
pub use crate::provider::Provider;
pub use crate::provider::RetryConfig;
pub use crate::provider::is_azure_responses_provider;
diff --git a/codex-rs/ext/image-generation/src/tests.rs b/codex-rs/ext/image-generation/src/tests.rs
index 0e6b9b278c..a2008d4e52 100644
--- a/codex-rs/ext/image-generation/src/tests.rs
+++ b/codex-rs/ext/image-generation/src/tests.rs
@@ -2,7 +2,6 @@ use codex_api::ImageBackground;
use codex_api::ImageEditRequest;
use codex_api::ImageGenerationRequest;
use codex_api::ImageQuality;
-use codex_api::ImageUrl;
use codex_extension_api::ToolOutput;
use codex_extension_api::ToolPayload;
use codex_extension_api::ToolSpec;
@@ -168,10 +167,9 @@ async fn recent_image_fallback_selects_newest_images_in_chronological_order() {
);
}
-/// A file-backed image in the requested window must not cause an older inline image to be edited.
#[tokio::test]
-async fn recent_image_fallback_rejects_file_backed_image_in_requested_window() {
- let error = request_for_call_args(
+async fn recent_image_fallback_passes_file_backed_image_to_edit_request() {
+ let request = request_for_call_args(
&ImagegenArgs {
prompt: "change the lighting".to_string(),
transparent_background: false,
@@ -196,19 +194,28 @@ async fn recent_image_fallback_rejects_file_backed_image_in_requested_window() {
&[],
)
.await
- .expect_err("a selected file-backed image should fail");
+ .expect("a selected file-backed image should be passed through");
assert_eq!(
- error.to_string(),
- "requested the last 1 conversation images, but that window includes a file-backed image \
- that cannot be used for editing"
+ request,
+ ImageRequest::Edit(ImageEditRequest {
+ images: vec![ImageReference::File {
+ file_id: "newer-file-backed-image".to_string(),
+ }],
+ prompt: "change the lighting".to_string(),
+ background: Some(ImageBackground::Opaque),
+ model: "gpt-image-2".to_string(),
+ n: None,
+ quality: Some(ImageQuality::Auto),
+ size: Some("auto".to_string()),
+ })
);
}
/// Tool-output file references must count toward the window instead of exposing an older image.
#[tokio::test]
-async fn recent_image_fallback_rejects_file_backed_tool_output_in_requested_window() {
- let error = request_for_call_args(
+async fn recent_image_fallback_passes_file_backed_tool_output_to_edit_request() {
+ let request = request_for_call_args(
&ImagegenArgs {
prompt: "change the lighting".to_string(),
transparent_background: false,
@@ -242,12 +249,21 @@ async fn recent_image_fallback_rejects_file_backed_tool_output_in_requested_wind
&[],
)
.await
- .expect_err("a selected file-backed tool output should fail");
+ .expect("a selected file-backed tool output should be passed through");
assert_eq!(
- error.to_string(),
- "requested the last 1 conversation images, but that window includes a file-backed image \
- that cannot be used for editing"
+ request,
+ ImageRequest::Edit(ImageEditRequest {
+ images: vec![ImageReference::File {
+ file_id: "newer-file-backed-image".to_string(),
+ }],
+ prompt: "change the lighting".to_string(),
+ background: Some(ImageBackground::Opaque),
+ model: "gpt-image-2".to_string(),
+ n: None,
+ quality: Some(ImageQuality::Auto),
+ size: Some("auto".to_string()),
+ })
);
}
@@ -432,7 +448,7 @@ fn expected_edit_request(prompt: &str, images: &[&str]) -> ImageEditRequest {
ImageEditRequest {
images: images
.iter()
- .map(|image| ImageUrl {
+ .map(|image| ImageReference::Inline {
image_url: format!("data:image/png;base64,{image}"),
})
.collect(),
diff --git a/codex-rs/ext/image-generation/src/tool.rs b/codex-rs/ext/image-generation/src/tool.rs
index fb54cb3595..f31a130ce3 100644
--- a/codex-rs/ext/image-generation/src/tool.rs
+++ b/codex-rs/ext/image-generation/src/tool.rs
@@ -6,7 +6,6 @@ use codex_api::ImageBackground;
use codex_api::ImageEditRequest;
use codex_api::ImageGenerationRequest;
use codex_api::ImageQuality;
-use codex_api::ImageUrl;
use codex_exec_server::CreateDirectoryOptions;
use codex_exec_server::LOCAL_FS;
use codex_extension_api::ExtensionTurnItem;
@@ -500,16 +499,14 @@ async fn request_for_call_args(
fn recent_images(
history: &[ResponseItem],
count: usize,
-) -> Result<Vec<ImageUrl>, FunctionCallError> {
+) -> Result<Vec<ImageReference>, FunctionCallError> {
let mut images = Vec::with_capacity(count);
'history: for item in history.iter().rev() {
- let mut image_urls = Vec::new();
+ let mut image_references = Vec::new();
match item {
ResponseItem::Message { content, .. } => {
- image_urls.extend(content.iter().rev().filter_map(|item| match item {
- ContentItem::InputImage { image, .. } => {
- Some(inline_image_url(image).map(str::to_owned))
- }
+ image_references.extend(content.iter().rev().filter_map(|item| match item {
+ ContentItem::InputImage { image, .. } => Some(image.clone()),
ContentItem::InputText { .. }
| ContentItem::InputAudio { .. }
| ContentItem::OutputText { .. } => None,
@@ -517,12 +514,12 @@ fn recent_images(
}
ResponseItem::FunctionCallOutput { output, .. }
| ResponseItem::CustomToolCallOutput { output, .. } => {
- image_urls.extend(
- output_images(output).map(|image| inline_image_url(image).map(str::to_owned)),
- );
+ image_references.extend(output_images(output).cloned());
}
ResponseItem::ImageGenerationCall { result, .. } if !result.is_empty() => {
- image_urls.push(Some(format!("data:image/png;base64,{result}")));
+ image_references.push(ImageReference::Inline {
+ image_url: format!("data:image/png;base64,{result}"),
+ });
}
ResponseItem::AdditionalTools { .. }
| ResponseItem::Reasoning { .. }
@@ -540,8 +537,8 @@ fn recent_images(
| ResponseItem::ContextCompaction { .. }
| ResponseItem::Other => {}
}
- for image_url in image_urls {
- images.push(image_url.map(|image_url| ImageUrl { image_url }));
+ for image in image_references {
+ images.push(image);
if images.len() == count {
break 'history;
}
@@ -553,18 +550,8 @@ fn recent_images(
images.len()
)));
}
- let mut inline_images = Vec::with_capacity(count);
- for image in images {
- let Some(image) = image else {
- return Err(FunctionCallError::RespondToModel(format!(
- "requested the last {count} conversation images, but that window includes a \
- file-backed image that cannot be used for editing"
- )));
- };
- inline_images.push(image);
- }
- inline_images.reverse();
- Ok(inline_images)
+ images.reverse();
+ Ok(images)
}
/// Extracts image references from a tool output in newest-first order.
@@ -582,18 +569,10 @@ fn output_images(output: &FunctionCallOutputPayload) -> impl Iterator<Item = &Im
})
}
-fn inline_image_url(image: &ImageReference) -> Option<&str> {
- match image {
- ImageReference::Inline { image_url } => Some(image_url),
- // TODO(kc) Image generation and the Images API only accept inline image URLs.
- ImageReference::File { .. } => None,
- }
-}
-
async fn image_url(
path: &AbsolutePathBuf,
environment: &ToolEnvironment<'_>,
-) -> Result<ImageUrl, FunctionCallError> {
+) -> Result<ImageReference, FunctionCallError> {
let path_uri = PathUri::from_abs_path(path);
let sandbox = environment.file_system_sandbox_context.clone();
let bytes = environment
@@ -614,7 +593,7 @@ async fn image_url(
))
},
)?;
- Ok(ImageUrl {
+ Ok(ImageReference::Inline {
image_url: image.into_data_url(),
})
}