Skip to content

armgit / codex

Public Imported from GitHub
0

Lightweight coding agent that runs in your terminal

Rust

Add explicit background control to image generation (#47484)

alicec-oai40eac3ce8a0c
diff --git a/codex-rs/app-server/tests/suite/v2/imagegen_extension.rs b/codex-rs/app-server/tests/suite/v2/imagegen_extension.rs
index b3e5e7d2aa..849a94f65d 100644
--- a/codex-rs/app-server/tests/suite/v2/imagegen_extension.rs
+++ b/codex-rs/app-server/tests/suite/v2/imagegen_extension.rs
@@ -28,6 +28,7 @@ use core_test_support::skip_if_remote;
use pretty_assertions::assert_eq;
use serde_json::json;
use tempfile::TempDir;
+use test_case::test_case;
use tokio::time::timeout;
use wiremock::Mock;
use wiremock::MockServer;
@@ -59,8 +60,14 @@ const DEFAULT_READ_TIMEOUT: Duration = Duration::from_secs(60);
#[cfg(not(any(target_os = "macos", windows)))]
const DEFAULT_READ_TIMEOUT: Duration = Duration::from_secs(10);
+#[test_case(Some(true), "transparent"; "transparent")]
+#[test_case(Some(false), "opaque"; "opaque")]
+#[test_case(None, "opaque"; "omitted_defaults_to_opaque")]
#[tokio::test]
-async fn standalone_image_generation_returns_saved_path_hint_to_model() -> Result<()> {
+async fn standalone_image_generation_returns_saved_path_hint_to_model(
+ requested_transparent_background: Option<bool>,
+ expected_background: &str,
+) -> Result<()> {
let call_id = "image-run-1";
let server = responses::start_mock_server().await;
Mock::given(method("POST"))
@@ -80,6 +87,7 @@ async fn standalone_image_generation_returns_saved_path_hint_to_model() -> Resul
.expect(1)
.mount(&server)
.await;
+ let arguments = imagegen_arguments("paint a blue whale", requested_transparent_background);
let response_mock = responses::mount_sse_sequence(
&server,
@@ -90,10 +98,7 @@ async fn standalone_image_generation_returns_saved_path_hint_to_model() -> Resul
call_id,
"image_gen",
"imagegen",
- &json!({
- "prompt": "paint a blue whale",
- })
- .to_string(),
+ &arguments.to_string(),
),
responses::ev_completed("resp-1"),
]),
@@ -158,6 +163,16 @@ async fn standalone_image_generation_returns_saved_path_hint_to_model() -> Resul
.into_iter()
.find(|request| request.url.path() == "/api/codex/images/generations")
.context("image generation request should be sent")?;
+ assert_eq!(
+ image_request.body_json::<serde_json::Value>()?,
+ json!({
+ "prompt": "paint a blue whale",
+ "background": expected_background,
+ "model": "gpt-image-2",
+ "quality": "auto",
+ "size": "auto",
+ })
+ );
assert_eq!(
image_request
.headers
@@ -231,7 +246,11 @@ async fn transparent_image_preserves_output_metadata_and_persisted_history() ->
call_id,
"image_gen",
"imagegen",
- &json!({"prompt": "a blue whale on a transparent background"}).to_string(),
+ &json!({
+ "prompt": "a blue whale on a transparent background",
+ "transparent_background": true,
+ })
+ .to_string(),
),
responses::ev_completed("resp-1"),
]),
@@ -390,6 +409,79 @@ async fn automatic_image_background_preserves_unknown_transparency() -> Result<(
Ok(())
}
+#[test_case(json!(null); "null")]
+#[test_case(json!("true"); "string")]
+#[test_case(json!(1); "number")]
+#[tokio::test]
+async fn invalid_transparent_background_does_not_dispatch(
+ transparent_background: serde_json::Value,
+) -> Result<()> {
+ let call_id = "invalid-background";
+ let server = responses::start_mock_server().await;
+ let response_mock = responses::mount_sse_sequence(
+ &server,
+ vec![
+ responses::sse(vec![
+ responses::ev_response_created("resp-1"),
+ responses::ev_function_call_with_namespace(
+ call_id,
+ "image_gen",
+ "imagegen",
+ &json!({
+ "prompt": "paint a blue whale",
+ "transparent_background": transparent_background,
+ })
+ .to_string(),
+ ),
+ responses::ev_completed("resp-1"),
+ ]),
+ responses::sse(vec![
+ responses::ev_assistant_message("msg-1", "The background argument was invalid."),
+ responses::ev_completed("resp-2"),
+ ]),
+ ],
+ )
+ .await;
+
+ let codex_home = TempDir::new()?;
+ create_config_toml(codex_home.path(), &server.uri(), ImagegenTestMode::Direct)?;
+ write_chatgpt_auth(
+ codex_home.path(),
+ ChatGptAuthFixture::new("access-chatgpt"),
+ AuthCredentialsStoreMode::File,
+ )?;
+ let mut mcp = TestAppServer::builder()
+ .with_codex_home(codex_home.path())
+ .with_env_overrides(&[("OPENAI_API_KEY", None)])
+ .build_initialized_with_timeout(DEFAULT_READ_TIMEOUT)
+ .await?;
+ start_image_generation_turn(&mut mcp, ThreadStartParams::default()).await?;
+ timeout(
+ DEFAULT_READ_TIMEOUT,
+ mcp.read_stream_until_notification_message("turn/completed"),
+ )
+ .await??;
+
+ let requests = response_mock.requests();
+ assert_eq!(requests.len(), 2);
+ let output = requests[1].function_call_output(call_id);
+ assert!(
+ output["output"]
+ .as_str()
+ .is_some_and(|text| text.contains("expected a boolean")),
+ "the model should receive the argument validation error: {output}"
+ );
+ assert!(
+ server
+ .received_requests()
+ .await
+ .context("failed to fetch received requests")?
+ .iter()
+ .all(|request| !request.url.path().starts_with("/api/codex/images/"))
+ );
+ Ok(())
+}
+
#[tokio::test]
async fn standalone_image_generation_failure_emits_terminal_item() -> Result<()> {
let call_id = "image-run-failed";
@@ -624,8 +716,14 @@ async fn image_generation_usage_limit_preserves_correlated_failure_metadata() ->
Ok(())
}
+#[test_case(Some(true), "transparent"; "transparent")]
+#[test_case(Some(false), "opaque"; "opaque")]
+#[test_case(None, "opaque"; "omitted_defaults_to_opaque")]
#[tokio::test]
-async fn standalone_image_edit_uses_attached_model_visible_image() -> Result<()> {
+async fn standalone_image_edit_uses_attached_model_visible_image(
+ requested_transparent_background: Option<bool>,
+ expected_background: &str,
+) -> Result<()> {
skip_if_remote!(
Ok(()),
"remote executors use different imagegen storage approaches, so host-local image paths are unavailable"
@@ -634,11 +732,10 @@ async fn standalone_image_edit_uses_attached_model_visible_image() -> Result<()>
let (edit_request, _) = run_image_edit_test(|codex_home| {
let image_path = codex_home.join("attached.png");
std::fs::write(&image_path, TINY_PNG_BYTES)?;
+ let mut arguments = imagegen_arguments("add a red hat", requested_transparent_background);
+ arguments["referenced_image_paths"] = json!([image_path.display().to_string()]);
Ok((
- json!({
- "prompt": "add a red hat",
- "referenced_image_paths": [image_path.display().to_string()],
- }),
+ arguments,
vec![
V2UserInput::Text {
text: "Edit the attached image".to_string(),
@@ -652,21 +749,35 @@ async fn standalone_image_edit_uses_attached_model_visible_image() -> Result<()>
))
})
.await?;
- assert_eq!(edit_request["prompt"], "add a red hat");
- assert_eq!(edit_request["images"][0]["image_url"], TINY_PNG_DATA_URL);
+ assert_eq!(
+ edit_request,
+ json!({
+ "prompt": "add a red hat",
+ "images": [{"image_url": TINY_PNG_DATA_URL}],
+ "background": expected_background,
+ "model": "gpt-image-2",
+ "quality": "auto",
+ "size": "auto",
+ })
+ );
Ok(())
}
+#[test_case(Some(true), "transparent"; "transparent")]
+#[test_case(Some(false), "opaque"; "opaque")]
+#[test_case(None, "opaque"; "omitted_defaults_to_opaque")]
#[tokio::test]
-async fn transparent_image_edit_preserves_metadata_and_recent_pathless_image() -> Result<()> {
+async fn transparent_image_edit_preserves_metadata_and_recent_pathless_image(
+ requested_transparent_background: Option<bool>,
+ expected_background: &str,
+) -> Result<()> {
let image_url = TINY_PNG_DATA_URL;
let (edit_request, completed_image) = run_image_edit_test(|_| {
+ let mut arguments = imagegen_arguments("add a red hat", requested_transparent_background);
+ arguments["num_last_images_to_include"] = json!(1);
Ok((
- json!({
- "prompt": "add a red hat",
- "num_last_images_to_include": 1,
- }),
+ arguments,
vec![
V2UserInput::Text {
text: "Edit the attached image".to_string(),
@@ -682,8 +793,17 @@ async fn transparent_image_edit_preserves_metadata_and_recent_pathless_image() -
))
})
.await?;
- assert_eq!(edit_request["prompt"], "add a red hat");
- assert_eq!(edit_request["images"][0]["image_url"], image_url);
+ assert_eq!(
+ edit_request,
+ json!({
+ "prompt": "add a red hat",
+ "images": [{"image_url": image_url}],
+ "background": expected_background,
+ "model": "gpt-image-2",
+ "quality": "auto",
+ "size": "auto",
+ })
+ );
assert_eq!(completed_image.transparent_background, Some(true));
Ok(())
@@ -735,11 +855,18 @@ async fn standalone_image_generation_is_exposed_in_code_mode_only() -> Result<()
}
#[cfg(not(windows))]
+#[test_case(Some(true), "transparent"; "transparent")]
+#[test_case(Some(false), "opaque"; "opaque")]
+#[test_case(None, "opaque"; "omitted_defaults_to_opaque")]
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-async fn standalone_image_generation_is_callable_from_code_mode_only() -> Result<()> {
+async fn standalone_image_generation_is_callable_from_code_mode_only(
+ requested_transparent_background: Option<bool>,
+ expected_background: &str,
+) -> Result<()> {
let call_id = "code-mode-image-run-1";
let server = responses::start_mock_server().await;
mount_image_response_with_background(&server, "opaque").await;
+ let arguments = imagegen_arguments("paint a blue whale", requested_transparent_background);
let response_mock = responses::mount_sse_sequence(
&server,
@@ -749,12 +876,12 @@ async fn standalone_image_generation_is_callable_from_code_mode_only() -> Result
responses::ev_custom_tool_call(
call_id,
"exec",
- r#"
-const result = await tools.image_gen__imagegen({
- prompt: "paint a blue whale",
-});
+ &format!(
+ r#"
+const result = await tools.image_gen__imagegen({arguments});
generatedImage(result);
-"#,
+"#
+ ),
),
responses::ev_completed("resp-1"),
]),
@@ -808,10 +935,35 @@ generatedImage(result);
.is_some_and(|text| text.contains("Generated images are saved"))
);
assert_eq!(output["output"].as_array().map(Vec::len), Some(3));
+ let image_request = server
+ .received_requests()
+ .await
+ .context("failed to fetch received requests")?
+ .into_iter()
+ .find(|request| request.url.path() == "/api/codex/images/generations")
+ .context("image generation request should be sent")?;
+ assert_eq!(
+ image_request.body_json::<serde_json::Value>()?,
+ json!({
+ "prompt": "paint a blue whale",
+ "background": expected_background,
+ "model": "gpt-image-2",
+ "quality": "auto",
+ "size": "auto",
+ })
+ );
Ok(())
}
+fn imagegen_arguments(prompt: &str, transparent_background: Option<bool>) -> serde_json::Value {
+ let mut arguments = json!({"prompt": prompt});
+ if let Some(transparent_background) = transparent_background {
+ arguments["transparent_background"] = json!(transparent_background);
+ }
+ arguments
+}
+
async fn start_image_generation_turn(
mcp: &mut TestAppServer,
thread_start_params: ThreadStartParams,
diff --git a/codex-rs/ext/image-generation/imagegen_description.md b/codex-rs/ext/image-generation/imagegen_description.md
index 0667fb494d..0e793f2a8d 100644
--- a/codex-rs/ext/image-generation/imagegen_description.md
+++ b/codex-rs/ext/image-generation/imagegen_description.md
@@ -6,6 +6,7 @@ The `image_gen.imagegen` tool enables image generation from descriptions and edi
Guidelines:
- imagegen needs a few minutes to finish. In code-mode, use the first-line @exec directive to give the initial call 120 seconds and the same yield for any waits that follow. Once it finishes, return the image with generatedImage(result).
- Avoid printing the full result or its base64 image data with `text()` or `notify()`; print only small metadata when needed.
+- Set `transparent_background` to true when the request calls for a transparent background, including background removal or a cutout; set it to false otherwise. For edits, preserve existing transparency unless the user asks to change it.
- Omit both `referenced_image_paths` and `num_last_images_to_include` when generating a brand new image.
- For edits, use `referenced_image_paths` when every target image has a local file path.
- If you have not seen a local image yet, use `view_image` to inspect it before editing.
diff --git a/codex-rs/ext/image-generation/src/tests.rs b/codex-rs/ext/image-generation/src/tests.rs
index 791244445e..0e6b9b278c 100644
--- a/codex-rs/ext/image-generation/src/tests.rs
+++ b/codex-rs/ext/image-generation/src/tests.rs
@@ -62,6 +62,7 @@ async fn omitted_references_generate_with_fixed_defaults() {
request_for_call_args(
&ImagegenArgs {
prompt: "paint a moonlit lake".to_string(),
+ transparent_background: false,
referenced_image_paths: None,
num_last_images_to_include: None,
},
@@ -72,7 +73,7 @@ async fn omitted_references_generate_with_fixed_defaults() {
.expect("generation request should build"),
ImageRequest::Generate(ImageGenerationRequest {
prompt: "paint a moonlit lake".to_string(),
- background: Some(ImageBackground::Auto),
+ background: Some(ImageBackground::Opaque),
model: "gpt-image-2".to_string(),
n: None,
quality: Some(ImageQuality::Auto),
@@ -151,6 +152,7 @@ async fn recent_image_fallback_selects_newest_images_in_chronological_order() {
request_for_call_args(
&ImagegenArgs {
prompt: "change the lighting".to_string(),
+ transparent_background: false,
referenced_image_paths: None,
num_last_images_to_include: Some(5),
},
@@ -172,6 +174,7 @@ async fn recent_image_fallback_rejects_file_backed_image_in_requested_window() {
let error = request_for_call_args(
&ImagegenArgs {
prompt: "change the lighting".to_string(),
+ transparent_background: false,
referenced_image_paths: None,
num_last_images_to_include: Some(1),
},
@@ -208,6 +211,7 @@ async fn recent_image_fallback_rejects_file_backed_tool_output_in_requested_wind
let error = request_for_call_args(
&ImagegenArgs {
prompt: "change the lighting".to_string(),
+ transparent_background: false,
referenced_image_paths: None,
num_last_images_to_include: Some(1),
},
@@ -252,6 +256,7 @@ async fn conflicting_image_selectors_return_tool_error() {
let error = request_for_call_args(
&ImagegenArgs {
prompt: "change the lighting".to_string(),
+ transparent_background: false,
referenced_image_paths: Some(vec![
"/tmp/image.png"
.try_into()
@@ -276,6 +281,7 @@ async fn too_many_referenced_image_paths_return_tool_error() {
let error = request_for_call_args(
&ImagegenArgs {
prompt: "change the lighting".to_string(),
+ transparent_background: false,
referenced_image_paths: Some(
(0..6)
.map(|index| {
@@ -304,6 +310,7 @@ async fn recent_image_fallback_requires_requested_count() {
let error = request_for_call_args(
&ImagegenArgs {
prompt: "change the lighting".to_string(),
+ transparent_background: false,
referenced_image_paths: None,
num_last_images_to_include: Some(2),
},
@@ -430,7 +437,7 @@ fn expected_edit_request(prompt: &str, images: &[&str]) -> ImageEditRequest {
})
.collect(),
prompt: prompt.to_string(),
- background: Some(ImageBackground::Auto),
+ background: Some(ImageBackground::Opaque),
model: "gpt-image-2".to_string(),
n: None,
quality: Some(ImageQuality::Auto),
diff --git a/codex-rs/ext/image-generation/src/tool.rs b/codex-rs/ext/image-generation/src/tool.rs
index 689708823d..fb54cb3595 100644
--- a/codex-rs/ext/image-generation/src/tool.rs
+++ b/codex-rs/ext/image-generation/src/tool.rs
@@ -89,6 +89,9 @@ impl ImageGenerationTool {
#[serde(deny_unknown_fields)]
struct ImagegenArgs {
prompt: String,
+ /// Whether the output should have a transparent background. Defaults to false.
+ #[serde(default)]
+ transparent_background: bool,
#[schemars(length(max = 5))]
referenced_image_paths: Option<Vec<AbsolutePathBuf>>,
#[schemars(range(min = 1, max = 5))]
@@ -120,7 +123,7 @@ impl<'call> ToolExecutor<ToolCall<'call>> for ImageGenerationTool {
ToolName::namespaced(IMAGE_GEN_NAMESPACE, IMAGEGEN_TOOL_NAME)
}
- /// Advertises the model contract: a rewritten prompt and optional edit references.
+ /// Advertises a rewritten prompt, background choice, and optional edit references.
fn spec(&self) -> ToolSpec {
imagegen_tool_spec()
}
@@ -427,6 +430,11 @@ async fn request_for_call_args(
history: &[ResponseItem],
environments: &[ToolEnvironment<'_>],
) -> Result<ImageRequest, FunctionCallError> {
+ let background = if args.transparent_background {
+ ImageBackground::Transparent
+ } else {
+ ImageBackground::Opaque
+ };
let paths = args.referenced_image_paths.as_deref().unwrap_or_default();
if paths.len() > MAX_EDIT_IMAGES {
return Err(FunctionCallError::RespondToModel(format!(
@@ -437,7 +445,7 @@ async fn request_for_call_args(
(true, None) => {
return Ok(ImageRequest::Generate(ImageGenerationRequest {
prompt: args.prompt.clone(),
- background: Some(ImageBackground::Auto),
+ background: Some(background),
model: IMAGE_MODEL.to_string(),
n: None,
quality: Some(ImageQuality::Auto),
@@ -481,7 +489,7 @@ async fn request_for_call_args(
Ok(ImageRequest::Edit(ImageEditRequest {
images,
prompt: args.prompt.clone(),
- background: Some(ImageBackground::Auto),
+ background: Some(background),
model: IMAGE_MODEL.to_string(),
n: None,
quality: Some(ImageQuality::Auto),