Lightweight coding agent that runs in your terminal
Rust
Add explicit background control to image generation (#47484)
alicec-oai40eac3ce8a0c
diff --git a/codex-rs/app-server/tests/suite/v2/imagegen_extension.rs b/codex-rs/app-server/tests/suite/v2/imagegen_extension.rsindex b3e5e7d2aa..849a94f65d 100644--- a/codex-rs/app-server/tests/suite/v2/imagegen_extension.rs+++ b/codex-rs/app-server/tests/suite/v2/imagegen_extension.rs@@ -28,6 +28,7 @@ use core_test_support::skip_if_remote;use pretty_assertions::assert_eq;use serde_json::json;use tempfile::TempDir;+use test_case::test_case;use tokio::time::timeout;use wiremock::Mock;use wiremock::MockServer;@@ -59,8 +60,14 @@ const DEFAULT_READ_TIMEOUT: Duration = Duration::from_secs(60);#[cfg(not(any(target_os = "macos", windows)))]const DEFAULT_READ_TIMEOUT: Duration = Duration::from_secs(10);+#[test_case(Some(true), "transparent"; "transparent")]+#[test_case(Some(false), "opaque"; "opaque")]+#[test_case(None, "opaque"; "omitted_defaults_to_opaque")]#[tokio::test]-async fn standalone_image_generation_returns_saved_path_hint_to_model() -> Result<()> {+async fn standalone_image_generation_returns_saved_path_hint_to_model(+ requested_transparent_background: Option<bool>,+ expected_background: &str,+) -> Result<()> {let call_id = "image-run-1";let server = responses::start_mock_server().await;Mock::given(method("POST"))@@ -80,6 +87,7 @@ async fn standalone_image_generation_returns_saved_path_hint_to_model() -> Resul.expect(1).mount(&server).await;+ let arguments = imagegen_arguments("paint a blue whale", requested_transparent_background);let response_mock = responses::mount_sse_sequence(&server,@@ -90,10 +98,7 @@ async fn standalone_image_generation_returns_saved_path_hint_to_model() -> Resulcall_id,"image_gen","imagegen",- &json!({- "prompt": "paint a blue whale",- })- .to_string(),+ &arguments.to_string(),),responses::ev_completed("resp-1"),]),@@ -158,6 +163,16 @@ async fn standalone_image_generation_returns_saved_path_hint_to_model() -> Resul.into_iter().find(|request| request.url.path() == "/api/codex/images/generations").context("image generation request should be sent")?;+ assert_eq!(+ image_request.body_json::<serde_json::Value>()?,+ json!({+ "prompt": "paint a blue whale",+ "background": expected_background,+ "model": "gpt-image-2",+ "quality": "auto",+ "size": "auto",+ })+ );assert_eq!(image_request.headers@@ -231,7 +246,11 @@ async fn transparent_image_preserves_output_metadata_and_persisted_history() ->call_id,"image_gen","imagegen",- &json!({"prompt": "a blue whale on a transparent background"}).to_string(),+ &json!({+ "prompt": "a blue whale on a transparent background",+ "transparent_background": true,+ })+ .to_string(),),responses::ev_completed("resp-1"),]),@@ -390,6 +409,79 @@ async fn automatic_image_background_preserves_unknown_transparency() -> Result<(Ok(())}+#[test_case(json!(null); "null")]+#[test_case(json!("true"); "string")]+#[test_case(json!(1); "number")]+#[tokio::test]+async fn invalid_transparent_background_does_not_dispatch(+ transparent_background: serde_json::Value,+) -> Result<()> {+ let call_id = "invalid-background";+ let server = responses::start_mock_server().await;+ let response_mock = responses::mount_sse_sequence(+ &server,+ vec![+ responses::sse(vec![+ responses::ev_response_created("resp-1"),+ responses::ev_function_call_with_namespace(+ call_id,+ "image_gen",+ "imagegen",+ &json!({+ "prompt": "paint a blue whale",+ "transparent_background": transparent_background,+ })+ .to_string(),+ ),+ responses::ev_completed("resp-1"),+ ]),+ responses::sse(vec![+ responses::ev_assistant_message("msg-1", "The background argument was invalid."),+ responses::ev_completed("resp-2"),+ ]),+ ],+ )+ .await;++ let codex_home = TempDir::new()?;+ create_config_toml(codex_home.path(), &server.uri(), ImagegenTestMode::Direct)?;+ write_chatgpt_auth(+ codex_home.path(),+ ChatGptAuthFixture::new("access-chatgpt"),+ AuthCredentialsStoreMode::File,+ )?;+ let mut mcp = TestAppServer::builder()+ .with_codex_home(codex_home.path())+ .with_env_overrides(&[("OPENAI_API_KEY", None)])+ .build_initialized_with_timeout(DEFAULT_READ_TIMEOUT)+ .await?;+ start_image_generation_turn(&mut mcp, ThreadStartParams::default()).await?;+ timeout(+ DEFAULT_READ_TIMEOUT,+ mcp.read_stream_until_notification_message("turn/completed"),+ )+ .await??;++ let requests = response_mock.requests();+ assert_eq!(requests.len(), 2);+ let output = requests[1].function_call_output(call_id);+ assert!(+ output["output"]+ .as_str()+ .is_some_and(|text| text.contains("expected a boolean")),+ "the model should receive the argument validation error: {output}"+ );+ assert!(+ server+ .received_requests()+ .await+ .context("failed to fetch received requests")?+ .iter()+ .all(|request| !request.url.path().starts_with("/api/codex/images/"))+ );+ Ok(())+}+#[tokio::test]async fn standalone_image_generation_failure_emits_terminal_item() -> Result<()> {let call_id = "image-run-failed";@@ -624,8 +716,14 @@ async fn image_generation_usage_limit_preserves_correlated_failure_metadata() ->Ok(())}+#[test_case(Some(true), "transparent"; "transparent")]+#[test_case(Some(false), "opaque"; "opaque")]+#[test_case(None, "opaque"; "omitted_defaults_to_opaque")]#[tokio::test]-async fn standalone_image_edit_uses_attached_model_visible_image() -> Result<()> {+async fn standalone_image_edit_uses_attached_model_visible_image(+ requested_transparent_background: Option<bool>,+ expected_background: &str,+) -> Result<()> {skip_if_remote!(Ok(()),"remote executors use different imagegen storage approaches, so host-local image paths are unavailable"@@ -634,11 +732,10 @@ async fn standalone_image_edit_uses_attached_model_visible_image() -> Result<()>let (edit_request, _) = run_image_edit_test(|codex_home| {let image_path = codex_home.join("attached.png");std::fs::write(&image_path, TINY_PNG_BYTES)?;+ let mut arguments = imagegen_arguments("add a red hat", requested_transparent_background);+ arguments["referenced_image_paths"] = json!([image_path.display().to_string()]);Ok((- json!({- "prompt": "add a red hat",- "referenced_image_paths": [image_path.display().to_string()],- }),+ arguments,vec![V2UserInput::Text {text: "Edit the attached image".to_string(),@@ -652,21 +749,35 @@ async fn standalone_image_edit_uses_attached_model_visible_image() -> Result<()>))}).await?;- assert_eq!(edit_request["prompt"], "add a red hat");- assert_eq!(edit_request["images"][0]["image_url"], TINY_PNG_DATA_URL);+ assert_eq!(+ edit_request,+ json!({+ "prompt": "add a red hat",+ "images": [{"image_url": TINY_PNG_DATA_URL}],+ "background": expected_background,+ "model": "gpt-image-2",+ "quality": "auto",+ "size": "auto",+ })+ );Ok(())}+#[test_case(Some(true), "transparent"; "transparent")]+#[test_case(Some(false), "opaque"; "opaque")]+#[test_case(None, "opaque"; "omitted_defaults_to_opaque")]#[tokio::test]-async fn transparent_image_edit_preserves_metadata_and_recent_pathless_image() -> Result<()> {+async fn transparent_image_edit_preserves_metadata_and_recent_pathless_image(+ requested_transparent_background: Option<bool>,+ expected_background: &str,+) -> Result<()> {let image_url = TINY_PNG_DATA_URL;let (edit_request, completed_image) = run_image_edit_test(|_| {+ let mut arguments = imagegen_arguments("add a red hat", requested_transparent_background);+ arguments["num_last_images_to_include"] = json!(1);Ok((- json!({- "prompt": "add a red hat",- "num_last_images_to_include": 1,- }),+ arguments,vec![V2UserInput::Text {text: "Edit the attached image".to_string(),@@ -682,8 +793,17 @@ async fn transparent_image_edit_preserves_metadata_and_recent_pathless_image() -))}).await?;- assert_eq!(edit_request["prompt"], "add a red hat");- assert_eq!(edit_request["images"][0]["image_url"], image_url);+ assert_eq!(+ edit_request,+ json!({+ "prompt": "add a red hat",+ "images": [{"image_url": image_url}],+ "background": expected_background,+ "model": "gpt-image-2",+ "quality": "auto",+ "size": "auto",+ })+ );assert_eq!(completed_image.transparent_background, Some(true));Ok(())@@ -735,11 +855,18 @@ async fn standalone_image_generation_is_exposed_in_code_mode_only() -> Result<()}#[cfg(not(windows))]+#[test_case(Some(true), "transparent"; "transparent")]+#[test_case(Some(false), "opaque"; "opaque")]+#[test_case(None, "opaque"; "omitted_defaults_to_opaque")]#[tokio::test(flavor = "multi_thread", worker_threads = 2)]-async fn standalone_image_generation_is_callable_from_code_mode_only() -> Result<()> {+async fn standalone_image_generation_is_callable_from_code_mode_only(+ requested_transparent_background: Option<bool>,+ expected_background: &str,+) -> Result<()> {let call_id = "code-mode-image-run-1";let server = responses::start_mock_server().await;mount_image_response_with_background(&server, "opaque").await;+ let arguments = imagegen_arguments("paint a blue whale", requested_transparent_background);let response_mock = responses::mount_sse_sequence(&server,@@ -749,12 +876,12 @@ async fn standalone_image_generation_is_callable_from_code_mode_only() -> Resultresponses::ev_custom_tool_call(call_id,"exec",- r#"-const result = await tools.image_gen__imagegen({- prompt: "paint a blue whale",-});+ &format!(+ r#"+const result = await tools.image_gen__imagegen({arguments});generatedImage(result);-"#,+"#+ ),),responses::ev_completed("resp-1"),]),@@ -808,10 +935,35 @@ generatedImage(result);.is_some_and(|text| text.contains("Generated images are saved")));assert_eq!(output["output"].as_array().map(Vec::len), Some(3));+ let image_request = server+ .received_requests()+ .await+ .context("failed to fetch received requests")?+ .into_iter()+ .find(|request| request.url.path() == "/api/codex/images/generations")+ .context("image generation request should be sent")?;+ assert_eq!(+ image_request.body_json::<serde_json::Value>()?,+ json!({+ "prompt": "paint a blue whale",+ "background": expected_background,+ "model": "gpt-image-2",+ "quality": "auto",+ "size": "auto",+ })+ );Ok(())}+fn imagegen_arguments(prompt: &str, transparent_background: Option<bool>) -> serde_json::Value {+ let mut arguments = json!({"prompt": prompt});+ if let Some(transparent_background) = transparent_background {+ arguments["transparent_background"] = json!(transparent_background);+ }+ arguments+}+async fn start_image_generation_turn(mcp: &mut TestAppServer,thread_start_params: ThreadStartParams,diff --git a/codex-rs/ext/image-generation/imagegen_description.md b/codex-rs/ext/image-generation/imagegen_description.mdindex 0667fb494d..0e793f2a8d 100644--- a/codex-rs/ext/image-generation/imagegen_description.md+++ b/codex-rs/ext/image-generation/imagegen_description.md@@ -6,6 +6,7 @@ The `image_gen.imagegen` tool enables image generation from descriptions and ediGuidelines:- imagegen needs a few minutes to finish. In code-mode, use the first-line @exec directive to give the initial call 120 seconds and the same yield for any waits that follow. Once it finishes, return the image with generatedImage(result).- Avoid printing the full result or its base64 image data with `text()` or `notify()`; print only small metadata when needed.+- Set `transparent_background` to true when the request calls for a transparent background, including background removal or a cutout; set it to false otherwise. For edits, preserve existing transparency unless the user asks to change it.- Omit both `referenced_image_paths` and `num_last_images_to_include` when generating a brand new image.- For edits, use `referenced_image_paths` when every target image has a local file path.- If you have not seen a local image yet, use `view_image` to inspect it before editing.diff --git a/codex-rs/ext/image-generation/src/tests.rs b/codex-rs/ext/image-generation/src/tests.rsindex 791244445e..0e6b9b278c 100644--- a/codex-rs/ext/image-generation/src/tests.rs+++ b/codex-rs/ext/image-generation/src/tests.rs@@ -62,6 +62,7 @@ async fn omitted_references_generate_with_fixed_defaults() {request_for_call_args(&ImagegenArgs {prompt: "paint a moonlit lake".to_string(),+ transparent_background: false,referenced_image_paths: None,num_last_images_to_include: None,},@@ -72,7 +73,7 @@ async fn omitted_references_generate_with_fixed_defaults() {.expect("generation request should build"),ImageRequest::Generate(ImageGenerationRequest {prompt: "paint a moonlit lake".to_string(),- background: Some(ImageBackground::Auto),+ background: Some(ImageBackground::Opaque),model: "gpt-image-2".to_string(),n: None,quality: Some(ImageQuality::Auto),@@ -151,6 +152,7 @@ async fn recent_image_fallback_selects_newest_images_in_chronological_order() {request_for_call_args(&ImagegenArgs {prompt: "change the lighting".to_string(),+ transparent_background: false,referenced_image_paths: None,num_last_images_to_include: Some(5),},@@ -172,6 +174,7 @@ async fn recent_image_fallback_rejects_file_backed_image_in_requested_window() {let error = request_for_call_args(&ImagegenArgs {prompt: "change the lighting".to_string(),+ transparent_background: false,referenced_image_paths: None,num_last_images_to_include: Some(1),},@@ -208,6 +211,7 @@ async fn recent_image_fallback_rejects_file_backed_tool_output_in_requested_windlet error = request_for_call_args(&ImagegenArgs {prompt: "change the lighting".to_string(),+ transparent_background: false,referenced_image_paths: None,num_last_images_to_include: Some(1),},@@ -252,6 +256,7 @@ async fn conflicting_image_selectors_return_tool_error() {let error = request_for_call_args(&ImagegenArgs {prompt: "change the lighting".to_string(),+ transparent_background: false,referenced_image_paths: Some(vec!["/tmp/image.png".try_into()@@ -276,6 +281,7 @@ async fn too_many_referenced_image_paths_return_tool_error() {let error = request_for_call_args(&ImagegenArgs {prompt: "change the lighting".to_string(),+ transparent_background: false,referenced_image_paths: Some((0..6).map(|index| {@@ -304,6 +310,7 @@ async fn recent_image_fallback_requires_requested_count() {let error = request_for_call_args(&ImagegenArgs {prompt: "change the lighting".to_string(),+ transparent_background: false,referenced_image_paths: None,num_last_images_to_include: Some(2),},@@ -430,7 +437,7 @@ fn expected_edit_request(prompt: &str, images: &[&str]) -> ImageEditRequest {}).collect(),prompt: prompt.to_string(),- background: Some(ImageBackground::Auto),+ background: Some(ImageBackground::Opaque),model: "gpt-image-2".to_string(),n: None,quality: Some(ImageQuality::Auto),diff --git a/codex-rs/ext/image-generation/src/tool.rs b/codex-rs/ext/image-generation/src/tool.rsindex 689708823d..fb54cb3595 100644--- a/codex-rs/ext/image-generation/src/tool.rs+++ b/codex-rs/ext/image-generation/src/tool.rs@@ -89,6 +89,9 @@ impl ImageGenerationTool {#[serde(deny_unknown_fields)]struct ImagegenArgs {prompt: String,+ /// Whether the output should have a transparent background. Defaults to false.+ #[serde(default)]+ transparent_background: bool,#[schemars(length(max = 5))]referenced_image_paths: Option<Vec<AbsolutePathBuf>>,#[schemars(range(min = 1, max = 5))]@@ -120,7 +123,7 @@ impl<'call> ToolExecutor<ToolCall<'call>> for ImageGenerationTool {ToolName::namespaced(IMAGE_GEN_NAMESPACE, IMAGEGEN_TOOL_NAME)}- /// Advertises the model contract: a rewritten prompt and optional edit references.+ /// Advertises a rewritten prompt, background choice, and optional edit references.fn spec(&self) -> ToolSpec {imagegen_tool_spec()}@@ -427,6 +430,11 @@ async fn request_for_call_args(history: &[ResponseItem],environments: &[ToolEnvironment<'_>],) -> Result<ImageRequest, FunctionCallError> {+ let background = if args.transparent_background {+ ImageBackground::Transparent+ } else {+ ImageBackground::Opaque+ };let paths = args.referenced_image_paths.as_deref().unwrap_or_default();if paths.len() > MAX_EDIT_IMAGES {return Err(FunctionCallError::RespondToModel(format!(@@ -437,7 +445,7 @@ async fn request_for_call_args((true, None) => {return Ok(ImageRequest::Generate(ImageGenerationRequest {prompt: args.prompt.clone(),- background: Some(ImageBackground::Auto),+ background: Some(background),model: IMAGE_MODEL.to_string(),n: None,quality: Some(ImageQuality::Auto),@@ -481,7 +489,7 @@ async fn request_for_call_args(Ok(ImageRequest::Edit(ImageEditRequest {images,prompt: args.prompt.clone(),- background: Some(ImageBackground::Auto),+ background: Some(background),model: IMAGE_MODEL.to_string(),n: None,quality: Some(ImageQuality::Auto),