Skip to content

armgit / codex

Public Imported from GitHub
0

Lightweight coding agent that runs in your terminal

Rust

Validate images before returning `view_image` output (#37892)

Vivian Fang7a18a5c5285a
diff --git a/codex-rs/core/src/tools/handlers/view_image.rs b/codex-rs/core/src/tools/handlers/view_image.rs
index ca32af5b1a..d1271202d0 100644
--- a/codex-rs/core/src/tools/handlers/view_image.rs
+++ b/codex-rs/core/src/tools/handlers/view_image.rs
@@ -12,6 +12,7 @@ use serde::Deserialize;
use crate::function_tool::FunctionCallError;
use crate::original_image_detail::can_request_original_image_detail;
+use crate::tools::context::ToolCallSource;
use crate::tools::context::ToolInvocation;
use crate::tools::context::ToolOutput;
use crate::tools::context::ToolPayload;
@@ -24,6 +25,8 @@ use crate::tools::registry::CoreToolRuntime;
use crate::tools::registry::ToolExecutor;
use codex_tools::ToolName;
use codex_tools::ToolSpec;
+use image::ImageFormat;
+use std::io::Cursor;
pub struct ViewImageHandler {
options: ViewImageToolOptions,
@@ -49,6 +52,10 @@ impl ViewImageHandler {
const VIEW_IMAGE_UNSUPPORTED_MESSAGE: &str =
"view_image is not allowed because you do not support image inputs";
+const VIEW_IMAGE_INVALID_MESSAGE: &str =
+ "unable to process image: invalid or unsupported image data";
+const VIEW_IMAGE_ENCODING_ERROR_MESSAGE: &str =
+ "unable to process image: could not encode validated image";
#[derive(Deserialize)]
struct ViewImageArgs {
@@ -104,6 +111,7 @@ impl ViewImageHandler {
step_context,
payload,
call_id,
+ source,
..
} = invocation;
@@ -173,6 +181,9 @@ impl ViewImageHandler {
"unable to read image at `{model_visible_path}`: {error}"
))
})?;
+ let decoded_image = image::load_from_memory(&file_bytes).map_err(|_| {
+ FunctionCallError::RespondToModel(VIEW_IMAGE_INVALID_MESSAGE.to_string())
+ })?;
let can_request_original_detail = can_request_original_image_detail(&turn.model_info);
let use_original_detail = self.options.unified_image_budget
@@ -183,8 +194,21 @@ impl ViewImageHandler {
DEFAULT_IMAGE_DETAIL
};
- // The history insertion path owns image decoding and resizing.
- let image_url = data_url_from_bytes("application/octet-stream", &file_bytes);
+ let image_url = if matches!(source, ToolCallSource::CodeMode { .. }) {
+ // Code mode must receive only image pixels, while keeping the original
+ // dimensions for centralized resizing and resize notices.
+ let mut sanitized_bytes = Vec::new();
+ decoded_image
+ .write_to(&mut Cursor::new(&mut sanitized_bytes), ImageFormat::Png)
+ .map_err(|_| {
+ FunctionCallError::RespondToModel(VIEW_IMAGE_ENCODING_ERROR_MESSAGE.to_string())
+ })?;
+ data_url_from_bytes("image/png", &sanitized_bytes)
+ } else {
+ // Direct tool calls retain image metadata for centralized preparation.
+ data_url_from_bytes("application/octet-stream", &file_bytes)
+ };
+ drop(decoded_image);
let item = TurnItem::ImageView(ImageViewItem {
id: call_id,
@@ -255,13 +279,14 @@ mod tests {
use crate::session::step_context::StepContext;
use crate::session::tests::make_session_and_context;
use crate::session::turn_context::TurnEnvironment;
- use crate::tools::context::ToolCallSource;
use crate::tools::context::ToolInvocation;
use crate::turn_diff_tracker::TurnDiffTracker;
use codex_protocol::models::PermissionProfile;
use codex_utils_absolute_path::AbsolutePathBuf;
use codex_utils_path_uri::PathUri;
use core_test_support::TempDirExt;
+ use image::ImageBuffer;
+ use image::Rgba;
use pretty_assertions::assert_eq;
use serde_json::json;
use std::sync::Arc;
@@ -284,6 +309,15 @@ mod tests {
));
}
+ fn tiny_png() -> Vec<u8> {
+ let image = ImageBuffer::from_pixel(1, 1, Rgba([255u8, 0, 0, 255]));
+ let mut bytes = Vec::new();
+ image
+ .write_to(&mut Cursor::new(&mut bytes), ImageFormat::Png)
+ .expect("encode test image");
+ bytes
+ }
+
#[test]
fn log_preview_omits_image_data() {
let output = ViewImageOutput {
@@ -324,7 +358,7 @@ mod tests {
replace_primary_environment_cwd(&mut turn, image_cwd.clone());
let image_path = image_cwd.join("image.png");
- std::fs::write(image_path.as_path(), b"not a real image").expect("write test image");
+ std::fs::write(image_path.as_path(), tiny_png()).expect("write test image");
Arc::make_mut(&mut turn.config)
.permissions
.set_permission_profile(PermissionProfile::Disabled)
@@ -400,7 +434,7 @@ mod tests {
replace_primary_environment_cwd(&mut turn, image_cwd.clone());
let image_path = image_cwd.join("image.png");
- std::fs::write(image_path.as_path(), b"not a real image").expect("write test image");
+ std::fs::write(image_path.as_path(), tiny_png()).expect("write test image");
let TurnEnvironmentState::Ready(environment) = &mut turn.environments.environments[0]
else {
panic!("primary environment should be ready");
@@ -427,4 +461,90 @@ mod tests {
result.expect("explicit high detail should be accepted");
}
+
+ #[tokio::test(flavor = "multi_thread")]
+ async fn handle_returns_png_image_for_code_mode() {
+ let (session, mut turn) = make_session_and_context().await;
+ let image_dir = tempfile::tempdir().expect("create image temp dir");
+ let image_cwd = image_dir.abs();
+
+ replace_primary_environment_cwd(&mut turn, image_cwd.clone());
+ let image_path = image_cwd.join("image.png");
+ std::fs::write(image_path.as_path(), tiny_png()).expect("write test image");
+ let TurnEnvironmentState::Ready(environment) = &mut turn.environments.environments[0]
+ else {
+ panic!("primary environment should be ready");
+ };
+ environment.config.permission_profile =
+ PermissionProfileSnapshot::legacy(PermissionProfile::Disabled);
+ let turn = Arc::new(turn);
+
+ let output = ViewImageHandler::default()
+ .handle(ToolInvocation {
+ session: Arc::new(session),
+ step_context: StepContext::for_test(Arc::clone(&turn)),
+ turn,
+ cancellation_token: tokio_util::sync::CancellationToken::new(),
+ tracker: Arc::new(Mutex::new(TurnDiffTracker::new())),
+ call_id: "call-view-image".to_string(),
+ tool_name: codex_tools::ToolName::plain("view_image"),
+ source: ToolCallSource::CodeMode {
+ cell_id: "cell-1".to_string(),
+ runtime_tool_call_id: "tool-1".to_string(),
+ },
+ payload: ToolPayload::Function {
+ arguments: json!({ "path": "image.png" }).to_string(),
+ },
+ })
+ .await
+ .expect("code mode image should be accepted");
+
+ let result = output.code_mode_result(&ToolPayload::Function {
+ arguments: "{}".to_string(),
+ });
+ let image_url = result["image_url"]
+ .as_str()
+ .expect("code mode image URL should be a string");
+ assert!(image_url.starts_with("data:image/png;base64,"));
+ }
+
+ #[tokio::test(flavor = "multi_thread")]
+ async fn handle_rejects_invalid_image_before_returning_output() {
+ let (session, mut turn) = make_session_and_context().await;
+ let image_dir = tempfile::tempdir().expect("create image temp dir");
+ let image_cwd = image_dir.abs();
+
+ replace_primary_environment_cwd(&mut turn, image_cwd.clone());
+ let image_path = image_cwd.join("not-an-image.txt");
+ std::fs::write(image_path.as_path(), b"arbitrary file contents")
+ .expect("write invalid image");
+ let TurnEnvironmentState::Ready(environment) = &mut turn.environments.environments[0]
+ else {
+ panic!("primary environment should be ready");
+ };
+ environment.config.permission_profile =
+ PermissionProfileSnapshot::legacy(PermissionProfile::Disabled);
+ let turn = Arc::new(turn);
+
+ let result = ViewImageHandler::default()
+ .handle(ToolInvocation {
+ session: Arc::new(session),
+ step_context: StepContext::for_test(Arc::clone(&turn)),
+ turn,
+ cancellation_token: tokio_util::sync::CancellationToken::new(),
+ tracker: Arc::new(Mutex::new(TurnDiffTracker::new())),
+ call_id: "call-view-image".to_string(),
+ tool_name: codex_tools::ToolName::plain("view_image"),
+ source: ToolCallSource::Direct,
+ payload: ToolPayload::Function {
+ arguments: json!({ "path": "not-an-image.txt" }).to_string(),
+ },
+ })
+ .await;
+
+ let Err(FunctionCallError::RespondToModel(message)) = result else {
+ panic!("expected invalid image error");
+ };
+ assert_eq!(message, VIEW_IMAGE_INVALID_MESSAGE);
+ }
}
diff --git a/codex-rs/core/tests/suite/view_image.rs b/codex-rs/core/tests/suite/view_image.rs
index ef12dd54c7..fe9928e198 100644
--- a/codex-rs/core/tests/suite/view_image.rs
+++ b/codex-rs/core/tests/suite/view_image.rs
@@ -1443,7 +1443,7 @@ async fn view_image_tool_errors_when_path_is_directory() -> anyhow::Result<()> {
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-async fn view_image_tool_turns_invalid_image_into_placeholder() -> anyhow::Result<()> {
+async fn view_image_tool_rejects_invalid_image_before_tool_output() -> anyhow::Result<()> {
skip_if_no_network!(Ok(()));
let server = start_mock_server().await;
@@ -1496,12 +1496,13 @@ async fn view_image_tool_turns_invalid_image_into_placeholder() -> anyhow::Resul
.await;
let request = second_mock.single_request();
+ let output_text = request
+ .function_call_output_content_and_success(call_id)
+ .and_then(|(content, _)| content)
+ .context("invalid view_image error text present")?;
assert_eq!(
- request.function_call_output(call_id).get("output"),
- Some(&serde_json::json!([{
- "type": "input_text",
- "text": "image content omitted because it could not be processed"
- }]))
+ output_text,
+ "unable to process image: invalid or unsupported image data"
);
Ok(())
}