Lightweight coding agent that runs in your terminal
Rust
Add a unified image budget (#37206)
Curtis 'Fjord' Hawthorne0a0ebb853551
diff --git a/codex-rs/code-mode-protocol/src/description.rs b/codex-rs/code-mode-protocol/src/description.rsindex f2c787aaaf..dde8ac2980 100644--- a/codex-rs/code-mode-protocol/src/description.rs+++ b/codex-rs/code-mode-protocol/src/description.rs@@ -9,6 +9,8 @@ use crate::PUBLIC_TOOL_NAME;const MAX_JS_SAFE_INTEGER: u64 = (1_u64 << 53) - 1;const DEFERRED_NESTED_TOOLS_GUIDANCE: &str = r#"Some deferred nested tools may be omitted from this description. They are still available on the global `tools` object and listed in `ALL_TOOLS`.To find one, filter `ALL_TOOLS` by `name` and `description`."#;+const LEGACY_IMAGE_HELPER_DESCRIPTION: &str = r#"`image(imageUrlOrItem: string | { image_url: string; detail?: "auto" | "low" | "high" | "original" | null } | ImageContent, detail?: "auto" | "low" | "high" | "original" | null)`: Appends an image item. `image_url` should be a base64-encoded `data:` URL. To forward an MCP tool image, pass an individual `ImageContent` block from `result.content`, for example `image(result.content[0])`. MCP image blocks may request detail with `_meta: { "codex/imageDetail": "original" }`. When provided, the second `detail` argument overrides any detail embedded in the first argument."#;+const UNIFIED_IMAGE_HELPER_DESCRIPTION: &str = r#"`image(imageUrlOrItem: string | { image_url: string } | ImageContent)`: Appends an image item. `image_url` should be a base64-encoded `data:` URL. To forward an MCP tool image, pass an individual `ImageContent` block from `result.content`, for example `image(result.content[0])`."#;const EXEC_DESCRIPTION_TEMPLATE: &str = r#"Run JavaScript code to orchestrate/compose tool calls- Evaluates the provided JavaScript code in a fresh V8 isolate as an async module.- All nested tools are available on the global `tools` object, for example `await tools.exec_command(...)`. Tool names are exposed as normalized JavaScript identifiers, for example `await tools.mcp__ologs__get_profile(...)`.@@ -249,18 +251,31 @@ pub fn is_code_mode_nested_tool(tool_name: &str) -> bool {tool_name != crate::PUBLIC_TOOL_NAME && tool_name != crate::WAIT_TOOL_NAME}+#[derive(Clone, Copy, Debug, Eq, PartialEq)]+pub enum ImageDetailVisibility {+ Visible,+ Hidden,+}+pub fn build_exec_tool_description(enabled_tools: &[ToolDefinition],deferred_tools: &[ToolDefinition],namespace_descriptions: &BTreeMap<String, ToolNamespaceDescription>,default_exec_yield_time_ms: u64,code_mode_only: bool,+ image_detail_visibility: ImageDetailVisibility,) -> String {let mut sections = Vec::new();sections.push(EXEC_DESCRIPTION_TEMPLATE.replace("Defaults to 10000 ms.",&format!("Defaults to {default_exec_yield_time_ms} ms."),));+ if image_detail_visibility == ImageDetailVisibility::Hidden {+ sections[0] = sections[0].replace(+ LEGACY_IMAGE_HELPER_DESCRIPTION,+ UNIFIED_IMAGE_HELPER_DESCRIPTION,+ );+ }if !deferred_tools.is_empty() {sections.push(DEFERRED_NESTED_TOOLS_GUIDANCE.to_string());}@@ -714,6 +729,7 @@ fn render_json_schema_literal(value: &JsonValue) -> String {#[cfg(test)]mod tests {use super::CodeModeToolKind;+ use super::ImageDetailVisibility;use super::ParsedExecSource;use super::ToolDefinition;use super::ToolNamespaceDescription;@@ -874,6 +890,7 @@ mod tests {&BTreeMap::new(),crate::DEFAULT_EXEC_YIELD_TIME_MS,/*code_mode_only*/ true,+ ImageDetailVisibility::Visible,);assert!(description.contains("### `foo`@@ -890,6 +907,7 @@ bar"&BTreeMap::new(),crate::DEFAULT_EXEC_YIELD_TIME_MS,/*code_mode_only*/ false,+ ImageDetailVisibility::Visible,);assert!(description.contains("`audio(audioUrlOrItem:"));assert!(description.contains("`setTimeout(callback: () => void, delayMs?: number)`"));@@ -944,6 +962,7 @@ bar"&namespace_descriptions,crate::DEFAULT_EXEC_YIELD_TIME_MS,/*code_mode_only*/ true,+ ImageDetailVisibility::Visible,);assert_eq!(description.matches("## mcp__sample").count(), 1);assert!(description.contains("## mcp__sample\nShared namespace guidance."));@@ -985,6 +1004,7 @@ bar"&namespace_descriptions,crate::DEFAULT_EXEC_YIELD_TIME_MS,/*code_mode_only*/ true,+ ImageDetailVisibility::Visible,);assert!(!description.contains("## mcp__sample"));@@ -1085,6 +1105,7 @@ bar"&BTreeMap::new(),crate::DEFAULT_EXEC_YIELD_TIME_MS,/*code_mode_only*/ true,+ ImageDetailVisibility::Visible,);assert_eq!(@@ -1121,6 +1142,7 @@ bar"&BTreeMap::new(),crate::DEFAULT_EXEC_YIELD_TIME_MS,/*code_mode_only*/ true,+ ImageDetailVisibility::Visible,);assert!(description.contains("Some deferred nested tools may be omitted"));@@ -1143,6 +1165,7 @@ bar"&BTreeMap::new(),crate::DEFAULT_EXEC_YIELD_TIME_MS,/*code_mode_only*/ false,+ ImageDetailVisibility::Visible,);assert!(description.contains("Some deferred nested tools may be omitted"));diff --git a/codex-rs/code-mode-protocol/src/lib.rs b/codex-rs/code-mode-protocol/src/lib.rsindex c7133f5228..3ec8c9ef1a 100644--- a/codex-rs/code-mode-protocol/src/lib.rs+++ b/codex-rs/code-mode-protocol/src/lib.rs@@ -7,6 +7,7 @@ mod session;pub use description::CODE_MODE_PRAGMA_PREFIX;pub use description::CodeModeToolKind;pub use description::EnabledToolMetadata;+pub use description::ImageDetailVisibility;pub use description::ToolDefinition;pub use description::ToolNamespaceDescription;pub use description::augment_tool_definition;diff --git a/codex-rs/core/config.schema.json b/codex-rs/core/config.schema.jsonindex fd463f4af1..9d67ab7a84 100644--- a/codex-rs/core/config.schema.json+++ b/codex-rs/core/config.schema.json@@ -756,6 +756,9 @@"unified_exec_zsh_fork": {"type": "boolean"},+ "unified_image_budget": {+ "type": "boolean"+ },"use_agent_identity": {"type": "boolean"},@@ -5387,6 +5390,9 @@"unified_exec_zsh_fork": {"type": "boolean"},+ "unified_image_budget": {+ "type": "boolean"+ },"use_agent_identity": {"type": "boolean"},diff --git a/codex-rs/core/src/image_preparation.rs b/codex-rs/core/src/image_preparation.rsindex 36f617b129..9c067565b0 100644--- a/codex-rs/core/src/image_preparation.rs+++ b/codex-rs/core/src/image_preparation.rs@@ -1,13 +1,17 @@+use crate::config::ManagedFeatures;use crate::context::ContextualUserFragment;use crate::context::ImageResizeNotice;use crate::context::ImageResizeNoticeSource;use crate::context::ResizedImage;+use crate::original_image_detail::can_request_original_image_detail;use codex_analytics::ImageDetailSetting;use codex_analytics::ImagePreparationMetadata;+use codex_features::Feature;use codex_protocol::models::ContentItem;use codex_protocol::models::FunctionCallOutputContentItem;use codex_protocol::models::ImageDetail;use codex_protocol::models::ResponseItem;+use codex_protocol::openai_models::ModelInfo;use codex_utils_image::ImageProcessingError;use codex_utils_image::PromptImageMode;use codex_utils_image::PromptImageResizeLimits;@@ -26,11 +30,25 @@ const HIGH_DETAIL_LIMITS: PromptImageResizeLimits = PromptImageResizeLimits {max_dimension: 2048,max_patches: 2_500,};-const ORIGINAL_DETAIL_LIMITS: PromptImageResizeLimits = PromptImageResizeLimits {+const UNIFIED_IMAGE_LIMITS: PromptImageResizeLimits = PromptImageResizeLimits {max_dimension: 6000,max_patches: 10_000,};+#[derive(Clone, Copy, Debug, Eq, PartialEq)]+pub(crate) enum ImagePreparationMode {+ DetailBased,+ UnifiedBudget,+}++pub(crate) fn unified_image_budget_enabled(+ features: &ManagedFeatures,+ model_info: &ModelInfo,+) -> bool {+ features.enabled(Feature::UnifiedImageBudget)+ && (model_info.use_responses_lite || can_request_original_image_detail(model_info))+}+#[derive(Clone, Copy, Debug)]struct ImageOrigin<'a> {message_role: Option<&'a str>,@@ -76,6 +94,7 @@ impl ImagePreparationError {pub(crate) fn prepare_response_items(items: &mut Vec<ResponseItem>,+ mode: ImagePreparationMode,resize_notice_mode: ImageResizeNoticeMode,) -> Vec<ImagePreparationMetadata> {let mut metadata = Vec::new();@@ -95,6 +114,7 @@ pub(crate) fn prepare_response_items(ImageResizeNoticeMode::Disabled},&mut metadata,+ mode,);(!resized_images.is_empty()).then(|| {ImageResizeNotice::new(ImageResizeNoticeSource::UserMessage, resized_images)@@ -114,6 +134,7 @@ pub(crate) fn prepare_response_items(},resize_notice_mode,&mut metadata,+ mode,);(!resized_images.is_empty()).then(|| {ImageResizeNotice::new(ImageResizeNoticeSource::ToolOutput, resized_images)@@ -148,6 +169,7 @@ fn prepare_message_content(origin: ImageOrigin<'_>,resize_notice_mode: ImageResizeNoticeMode,metadata: &mut Vec<ImagePreparationMetadata>,+ mode: ImagePreparationMode,) -> Vec<ResizedImage> {let image_count = items.iter()@@ -158,7 +180,7 @@ fn prepare_message_content(for item in items {if let ContentItem::InputImage { image_url, detail } = item {image_number += 1;- match prepare_image(image_url, *detail, origin, metadata) {+ match prepare_image(image_url, detail, origin, metadata, mode) {Ok(Some(resize)) if resize_notice_mode == ImageResizeNoticeMode::Enabled => {resized_images.push(ResizedImage {image_number,@@ -187,6 +209,7 @@ fn prepare_tool_output_content(origin: ImageOrigin<'_>,resize_notice_mode: ImageResizeNoticeMode,metadata: &mut Vec<ImagePreparationMetadata>,+ mode: ImagePreparationMode,) -> Vec<ResizedImage> {let image_count = items.iter()@@ -197,7 +220,7 @@ fn prepare_tool_output_content(for item in items {if let FunctionCallOutputContentItem::InputImage { image_url, detail } = item {image_number += 1;- match prepare_image(image_url, *detail, origin, metadata) {+ match prepare_image(image_url, detail, origin, metadata, mode) {Ok(Some(resize)) if resize_notice_mode == ImageResizeNoticeMode::Enabled => {resized_images.push(ResizedImage {image_number,@@ -235,9 +258,10 @@ fn is_data_url(image_url: &str) -> bool {fn prepare_image(image_url: &mut String,- detail: Option<ImageDetail>,+ detail: &mut Option<ImageDetail>,origin: ImageOrigin<'_>,metadata: &mut Vec<ImagePreparationMetadata>,+ mode: ImagePreparationMode,) -> Result<Option<PreparedImageResize>, ImagePreparationError> {if is_remote_image_url(image_url) {return Err(ImagePreparationError::RemoteUrlUnsupported);@@ -246,12 +270,15 @@ fn prepare_image(return Ok(None);}- let (effective_detail, limits) = match detail {- None | Some(ImageDetail::Auto | ImageDetail::High) => {- (ImageDetailSetting::High, HIGH_DETAIL_LIMITS)- }- Some(ImageDetail::Original) => (ImageDetailSetting::Original, ORIGINAL_DETAIL_LIMITS),- Some(ImageDetail::Low) => return Err(ImagePreparationError::UnsupportedLowDetail),+ let (effective_detail, limits) = match mode {+ ImagePreparationMode::UnifiedBudget => (ImageDetailSetting::Original, UNIFIED_IMAGE_LIMITS),+ ImagePreparationMode::DetailBased => match detail {+ None | Some(ImageDetail::Auto | ImageDetail::High) => {+ (ImageDetailSetting::High, HIGH_DETAIL_LIMITS)+ }+ Some(ImageDetail::Original) => (ImageDetailSetting::Original, UNIFIED_IMAGE_LIMITS),+ Some(ImageDetail::Low) => return Err(ImagePreparationError::UnsupportedLowDetail),+ },};let image = load_data_url_for_prompt(image_url, PromptImageMode::ResizeWithLimits(limits))?;metadata.push(ImagePreparationMetadata {@@ -271,6 +298,11 @@ fn prepare_image(prepared_height: image.height,});*image_url = image.into_data_url();+ if mode == ImagePreparationMode::UnifiedBudget {+ // Preserve accurate context-window accounting while older transports still require an+ // image detail field. Responses Lite removes this compatibility hint before sending.+ *detail = Some(ImageDetail::Original);+ }Ok(resize)}diff --git a/codex-rs/core/src/image_preparation_tests.rs b/codex-rs/core/src/image_preparation_tests.rsindex 53fad8ff66..9919e41e3e 100644--- a/codex-rs/core/src/image_preparation_tests.rs+++ b/codex-rs/core/src/image_preparation_tests.rs@@ -51,7 +51,11 @@ fn preparation_preserves_small_image_bytes_and_replaces_remote_urls() {internal_chat_message_metadata_passthrough: None,}];- prepare_response_items(&mut items, ImageResizeNoticeMode::Disabled);+ prepare_response_items(+ &mut items,+ ImagePreparationMode::DetailBased,+ ImageResizeNoticeMode::Disabled,+ );let ResponseItem::Message { content, .. } = &items[0] else {panic!("expected message");@@ -105,7 +109,11 @@ fn detail_policies_apply_the_expected_budgets() {internal_chat_message_metadata_passthrough: None,}];- let metadata = prepare_response_items(&mut items, ImageResizeNoticeMode::Disabled);+ let metadata = prepare_response_items(+ &mut items,+ ImagePreparationMode::DetailBased,+ ImageResizeNoticeMode::Disabled,+ );let ResponseItem::Message { content, .. } = &items[0] else {panic!("expected message");@@ -144,7 +152,11 @@ fn preparation_reports_tool_output_item_id() {]),internal_chat_message_metadata_passthrough: None,}];- let metadata = prepare_response_items(&mut items, ImageResizeNoticeMode::Disabled);+ let metadata = prepare_response_items(+ &mut items,+ ImagePreparationMode::DetailBased,+ ImageResizeNoticeMode::Disabled,+ );assert_eq!(metadata,@@ -202,7 +214,11 @@ fn resize_notices_preserve_original_image_positions_and_skip_failed_images() {},];- prepare_response_items(&mut items, ImageResizeNoticeMode::Enabled);+ prepare_response_items(+ &mut items,+ ImagePreparationMode::DetailBased,+ ImageResizeNoticeMode::Enabled,+ );let expected_user_notice = concat!("<image_resize_notice>\n","Image 3 of 3 in the preceding user message was resized from 2048x2048 to 1600x1600 pixels.\n",@@ -325,7 +341,11 @@ fn preparation_replaces_only_failed_tool_images_and_preserves_metadata() {internal_chat_message_metadata_passthrough: None,}];- prepare_response_items(&mut items, ImageResizeNoticeMode::Disabled);+ prepare_response_items(+ &mut items,+ ImagePreparationMode::DetailBased,+ ImageResizeNoticeMode::Disabled,+ );assert_eq!(items,diff --git a/codex-rs/core/src/session/mod.rs b/codex-rs/core/src/session/mod.rsindex 01c742be89..96021f37a3 100644--- a/codex-rs/core/src/session/mod.rs+++ b/codex-rs/core/src/session/mod.rs@@ -31,8 +31,10 @@ use crate::environment_selection::TurnEnvironmentSnapshot;use crate::exec_policy::BANNED_PREFIX_SUGGESTIONS;use crate::exec_policy::ExecPolicyManager;use crate::exec_policy::default_policy_path;+use crate::image_preparation::ImagePreparationMode;use crate::image_preparation::ImageResizeNoticeMode;use crate::image_preparation::prepare_response_items as prepare_image_response_items;+use crate::image_preparation::unified_image_budget_enabled;use crate::parse_turn_item;use crate::realtime_conversation::RealtimeConversationManager;use crate::session::step_context::StepContext;@@ -1445,7 +1447,11 @@ impl Session {// will be processed again if the rollout is reconstructed in a future session.// Never backfill resize notices during replay; only newly recorded items may// emit them, so the historical model prefix remains unchanged.- let _ = prepare_image_response_items(&mut history, ImageResizeNoticeMode::Disabled);+ let _ = prepare_image_response_items(+ &mut history,+ ImagePreparationMode::DetailBased,+ ImageResizeNoticeMode::Disabled,+ );prepare_audio_response_items(&mut history);{let mut state = self.state.lock().await;@@ -2911,17 +2917,27 @@ impl Session {items: &'a [ResponseItem],) -> (Cow<'a, [ResponseItem]>, Vec<ImagePreparationMetadata>) {let mut items = items.to_vec();+ let image_preparation_mode = if unified_image_budget_enabled(+ &turn_context.config.features,+ &turn_context.model_info,+ ) {+ ImagePreparationMode::UnifiedBudget+ } else {+ ImagePreparationMode::DetailBased+ };+ let image_resize_notice_mode = if turn_context+ .config+ .features+ .enabled(Feature::ImageResizeNotice)+ {+ ImageResizeNoticeMode::Enabled+ } else {+ ImageResizeNoticeMode::Disabled+ };let image_preparations = prepare_image_response_items(&mut items,- if turn_context- .config- .features- .enabled(Feature::ImageResizeNotice)- {- ImageResizeNoticeMode::Enabled- } else {- ImageResizeNoticeMode::Disabled- },+ image_preparation_mode,+ image_resize_notice_mode,);prepare_audio_response_items(&mut items);// Most response items get their passthrough turn ID at the durable history boundary.diff --git a/codex-rs/core/src/tools/code_mode/execute_spec.rs b/codex-rs/core/src/tools/code_mode/execute_spec.rsindex b3476720c8..3b44117b91 100644--- a/codex-rs/core/src/tools/code_mode/execute_spec.rs+++ b/codex-rs/core/src/tools/code_mode/execute_spec.rs@@ -1,3 +1,4 @@+use codex_code_mode::ImageDetailVisibility;use codex_code_mode::ToolDefinition as CodeModeToolDefinition;use codex_tools::FreeformTool;use codex_tools::FreeformToolFormat;@@ -10,6 +11,7 @@ pub(crate) fn create_code_mode_tool(namespace_descriptions: &BTreeMap<String, codex_code_mode::ToolNamespaceDescription>,default_exec_yield_time_ms: u64,code_mode_only: bool,+ image_detail_visibility: ImageDetailVisibility,) -> ToolSpec {const CODE_MODE_FREEFORM_GRAMMAR: &str = r#"start: pragma_source | plain_source@@ -29,6 +31,7 @@ SOURCE: /[\s\S]+/namespace_descriptions,default_exec_yield_time_ms,code_mode_only,+ image_detail_visibility,),defer_loading: None,format: FreeformToolFormat {@@ -63,6 +66,7 @@ mod tests {&BTreeMap::new(),codex_code_mode::DEFAULT_EXEC_YIELD_TIME_MS,/*code_mode_only*/ true,+ ImageDetailVisibility::Visible,),ToolSpec::Freeform(FreeformTool {name: codex_code_mode::PUBLIC_TOOL_NAME.to_string(),@@ -72,6 +76,7 @@ mod tests {&BTreeMap::new(),codex_code_mode::DEFAULT_EXEC_YIELD_TIME_MS,/*code_mode_only*/ true,+ ImageDetailVisibility::Visible,),defer_loading: None,format: FreeformToolFormat {diff --git a/codex-rs/core/src/tools/handlers/view_image.rs b/codex-rs/core/src/tools/handlers/view_image.rsindex 9511f5d1a3..ca32af5b1a 100644--- a/codex-rs/core/src/tools/handlers/view_image.rs+++ b/codex-rs/core/src/tools/handlers/view_image.rs@@ -34,6 +34,7 @@ impl Default for ViewImageHandler {Self {options: ViewImageToolOptions {can_request_original_image_detail: false,+ unified_image_budget: false,include_environment_id: false,},}@@ -120,8 +121,7 @@ impl ViewImageHandler {environment_id,detail,} = parse_arguments(&arguments)?;- // `high` is the explicit spelling of the default resized path.- // Other string values remain invalid rather than being silently reinterpreted.+ // Keep accepting previously supported detail hints after they disappear from the schema.let detail = match detail.as_deref() {None => None,Some("high") => Some(ViewImageDetail::High),@@ -175,8 +175,8 @@ impl ViewImageHandler {})?;let can_request_original_detail = can_request_original_image_detail(&turn.model_info);- let use_original_detail =- can_request_original_detail && matches!(detail, Some(ViewImageDetail::Original));+ let use_original_detail = self.options.unified_image_budget+ || can_request_original_detail && matches!(detail, Some(ViewImageDetail::Original));let image_detail = if use_original_detail {ImageDetail::Original} else {@@ -196,6 +196,7 @@ impl ViewImageHandler {Ok(boxed_tool_output(ViewImageOutput {image_url,image_detail,+ unified_image_budget: self.options.unified_image_budget,}))}}@@ -205,6 +206,7 @@ impl CoreToolRuntime for ViewImageHandler {}pub struct ViewImageOutput {image_url: String,image_detail: ImageDetail,+ unified_image_budget: bool,}impl ToolOutput for ViewImageOutput {@@ -234,10 +236,14 @@ impl ToolOutput for ViewImageOutput {}fn code_mode_result(&self, _payload: &ToolPayload) -> serde_json::Value {- serde_json::json!({- "image_url": self.image_url,- "detail": self.image_detail- })+ if self.unified_image_budget {+ serde_json::json!({ "image_url": self.image_url })+ } else {+ serde_json::json!({+ "image_url": self.image_url,+ "detail": self.image_detail+ })+ }}}@@ -283,6 +289,7 @@ mod tests {let output = ViewImageOutput {image_url: "data:image/png;base64,AAA".to_string(),image_detail: DEFAULT_IMAGE_DETAIL,+ unified_image_budget: false,};assert_eq!(output.log_preview(), "<image data URL omitted: 25 bytes>");@@ -293,6 +300,7 @@ mod tests {let output = ViewImageOutput {image_url: "data:image/png;base64,AAA".to_string(),image_detail: DEFAULT_IMAGE_DETAIL,+ unified_image_budget: false,};let result = output.code_mode_result(&ToolPayload::Function {diff --git a/codex-rs/core/src/tools/handlers/view_image_spec.rs b/codex-rs/core/src/tools/handlers/view_image_spec.rsindex 5d2b7e0b15..f28d9f8b1a 100644--- a/codex-rs/core/src/tools/handlers/view_image_spec.rs+++ b/codex-rs/core/src/tools/handlers/view_image_spec.rs@@ -9,6 +9,7 @@ use std::collections::BTreeMap;#[derive(Debug, Clone, Copy, PartialEq, Eq)]pub struct ViewImageToolOptions {pub can_request_original_image_detail: bool,+ pub unified_image_budget: bool,pub include_environment_id: bool,}@@ -17,7 +18,7 @@ pub fn create_view_image_tool(options: ViewImageToolOptions) -> ToolSpec {"path".to_string(),JsonSchema::string(Some("Local filesystem path to an image file.".to_string())),)]);- if options.can_request_original_image_detail {+ if options.can_request_original_image_detail && !options.unified_image_budget {properties.insert("detail".to_string(),JsonSchema::string_enum(@@ -45,25 +46,29 @@ pub fn create_view_image_tool(options: ViewImageToolOptions) -> ToolSpec {strict: false,defer_loading: None,parameters: JsonSchema::object(properties, Some(vec!["path".to_string()]), Some(false.into())),- output_schema: Some(view_image_output_schema()),+ output_schema: Some(view_image_output_schema(options)),})}-fn view_image_output_schema() -> Value {- json!({+fn view_image_output_schema(options: ViewImageToolOptions) -> Value {+ let mut schema = json!({"type": "object","properties": {"image_url": {"type": "string","description": "Data URL for the loaded image."- },- "detail": {- "type": "string",- "enum": ["high", "original"],- "description": "Image detail hint returned by view_image. Returns `high` for default resized behavior or `original` when original resolution is preserved."}},- "required": ["image_url", "detail"],+ "required": ["image_url"],"additionalProperties": false- })+ });+ if !options.unified_image_budget {+ schema["properties"]["detail"] = json!({+ "type": "string",+ "enum": ["high", "original"],+ "description": "Image detail hint returned by view_image. Returns `high` for default resized behavior or `original` when original resolution is preserved."+ });+ schema["required"] = json!(["image_url", "detail"]);+ }+ schema}diff --git a/codex-rs/core/src/tools/spec_plan.rs b/codex-rs/core/src/tools/spec_plan.rsindex 4c13de5692..0e94988ec1 100644--- a/codex-rs/core/src/tools/spec_plan.rs+++ b/codex-rs/core/src/tools/spec_plan.rs@@ -1,6 +1,7 @@use crate::agent::exceeds_thread_spawn_depth_limit;use crate::agent::next_thread_spawn_depth;use crate::environment_selection::TurnEnvironmentSnapshot;+use crate::image_preparation::unified_image_budget_enabled;use crate::mcp_tool_exposure::append_mcp_tools;use crate::session::session::Session;use crate::session::turn_context::TurnContext;@@ -730,6 +731,12 @@ fn register_code_mode_executors(&namespace_descriptions,default_exec_yield_time_ms,tool_mode == ToolMode::CodeModeOnly,+ if unified_image_budget_enabled(&turn_context.config.features, &turn_context.model_info)+ {+ codex_code_mode::ImageDetailVisibility::Hidden+ } else {+ codex_code_mode::ImageDetailVisibility::Visible+ },),code_mode_nested_tool_specs,);@@ -838,6 +845,10 @@ fn add_core_tool_sources(context: &CoreToolPlanContext<'_>, registry: &mut ToolRcan_request_original_image_detail: can_request_original_image_detail(&turn_context.model_info,),+ unified_image_budget: unified_image_budget_enabled(+ &turn_context.config.features,+ &turn_context.model_info,+ ),include_environment_id,}));}@@ -1034,6 +1045,10 @@ fn add_core_utility_tools(context: &CoreToolPlanContext<'_>, registry: &mut Toolcan_request_original_image_detail: can_request_original_image_detail(&turn_context.model_info,),+ unified_image_budget: unified_image_budget_enabled(+ &turn_context.config.features,+ &turn_context.model_info,+ ),include_environment_id,}));}diff --git a/codex-rs/core/tests/suite/code_mode.rs b/codex-rs/core/tests/suite/code_mode.rsindex dc49f84919..9e3ade1f61 100644--- a/codex-rs/core/tests/suite/code_mode.rs+++ b/codex-rs/core/tests/suite/code_mode.rs@@ -82,6 +82,7 @@ use std::path::Path;use std::sync::Arc;use std::time::Duration;use std::time::Instant;+use test_case::test_case;use wiremock::Mock;use wiremock::MockServer;use wiremock::ResponseTemplate;@@ -3635,6 +3636,114 @@ async fn code_mode_resizes_explicit_original_image() -> Result<()> {Ok(())}+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]+async fn code_mode_unified_image_budget_accepts_legacy_detail_hints() -> Result<()> {+ skip_if_no_network!(Ok(()));++ let image = ImageBuffer::from_pixel(+ /*width*/ 2304,+ /*height*/ 864,+ Rgba([20, 40, 60, 255]),+ );+ let mut encoded = Cursor::new(Vec::new());+ DynamicImage::ImageRgba8(image).write_to(&mut encoded, image::ImageFormat::Png)?;+ let image_data = BASE64_STANDARD.encode(encoded.into_inner());+ let code = format!(+ r#"+const data = {};+const imageUrl = `data:image/png;base64,${{data}}`;+image(imageUrl);+image(imageUrl, "auto");+image(imageUrl, "high");+image({{ image_url: imageUrl, detail: "low" }});+image({{+ type: "image",+ data,+ mimeType: "image/png",+ _meta: {{ "codex/imageDetail": "original" }}+}});+"#,+ serde_json::to_string(&image_data)?+ );++ let server = responses::start_mock_server().await;+ let (_test, second_mock) = run_code_mode_turn_with_model_and_config(+ &server,+ "emit images with legacy detail arguments and MCP metadata",+ &code,+ "gpt-5.4",+ |config| {+ let _ = config.features.enable(Feature::UnifiedImageBudget);+ },+ )+ .await?;++ let request = second_mock.single_request();+ let items = custom_tool_output_items(&request, "call-1");+ assert_eq!(items.len(), 6, "unexpected code-mode output: {items:?}");+ for item in &items[1..] {+ assert_eq!(item["type"], "input_image");+ assert_eq!(item["detail"], "original");+ let image_url = item["image_url"]+ .as_str()+ .expect("image helper should emit a data URL");+ let (_, payload) = image_url+ .split_once(',')+ .expect("image data URL should have a base64 payload");+ let image = image::load_from_memory(&BASE64_STANDARD.decode(payload)?)?;+ assert_eq!(image.dimensions(), (2304, 864));+ }++ let body = request.body_json();+ let exec_description = body["tools"]+ .as_array()+ .and_then(|tools| tools.iter().find(|tool| tool["name"] == "exec"))+ .and_then(|tool| tool["description"].as_str())+ .expect("the model request should contain the code-mode exec tool");+ assert!(+ exec_description+ .contains("`image(imageUrlOrItem: string | { image_url: string } | ImageContent)`")+ );+ assert!(!exec_description.contains("codex/imageDetail"));+ assert!(!exec_description.contains("detail?:"));++ Ok(())+}++#[tokio::test(flavor = "multi_thread", worker_threads = 2)]+async fn code_mode_unified_image_budget_preserves_legacy_contract_for_unsupported_model()+-> Result<()> {+ skip_if_no_network!(Ok(()));++ let server = responses::start_mock_server().await;+ let (_test, second_mock) = run_code_mode_turn_with_model_and_config(+ &server,+ "emit an image on a legacy model",+ r#"image("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGP4z8DwHwAFAAH/iZk9HQAAAABJRU5ErkJggg==");"#,+ "gpt-5.2",+ |config| {+ let _ = config.features.enable(Feature::UnifiedImageBudget);+ },+ )+ .await?;++ let request = second_mock.single_request();+ let items = custom_tool_output_items(&request, "call-1");+ assert_eq!(items.len(), 2);+ assert_eq!(items[1]["detail"], "high");++ let body = request.body_json();+ let exec_description = body["tools"]+ .as_array()+ .and_then(|tools| tools.iter().find(|tool| tool["name"] == "exec"))+ .and_then(|tool| tool["description"].as_str())+ .expect("the model request should contain the code-mode exec tool");+ assert!(exec_description.contains("codex/imageDetail"));+ assert!(exec_description.contains("detail?:"));++ Ok(())+}+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]async fn code_mode_image_helper_rejects_remote_url() -> Result<()> {skip_if_no_network!(Ok(()));@@ -3718,8 +3827,12 @@ image(s.trim(), "original");Ok(())}+#[test_case(false; "legacy detail")]+#[test_case(true; "unified image budget")]#[tokio::test(flavor = "multi_thread", worker_threads = 2)]-async fn code_mode_can_use_view_image_result_with_image_helper() -> Result<()> {+async fn code_mode_can_use_view_image_result_with_image_helper(+ unified_image_budget: bool,+) -> Result<()> {skip_if_no_network!(Ok(()));let server = responses::start_mock_server().await;@@ -3727,6 +3840,9 @@ async fn code_mode_can_use_view_image_result_with_image_helper() -> Result<()> {.with_model("gpt-5.4").with_config(move |config| {let _ = config.features.enable(Feature::CodeMode);+ if unified_image_budget {+ let _ = config.features.enable(Feature::UnifiedImageBudget);+ }});let test = builder.build(&server).await?;@@ -3737,9 +3853,18 @@ async fn code_mode_can_use_view_image_result_with_image_helper() -> Result<()> {fs::write(&image_path, image_bytes)?;let image_path_json = serde_json::to_string(&image_path.to_string_lossy().to_string())?;+ let expected_output_keys = if unified_image_budget {+ r#"["image_url"]"#+ } else {+ r#"["detail","image_url"]"#+ };let code = format!(r#"const out = await tools.view_image({{ path: {image_path_json}, detail: "original" }});+const keys = Object.keys(out).sort();+if (JSON.stringify(keys) !== JSON.stringify({expected_output_keys})) {{+ throw new Error(`unexpected view_image output keys: ${{JSON.stringify(keys)}}`);+}}image(out);"#);diff --git a/codex-rs/core/tests/suite/view_image.rs b/codex-rs/core/tests/suite/view_image.rsindex be4083c094..ef12dd54c7 100644--- a/codex-rs/core/tests/suite/view_image.rs+++ b/codex-rs/core/tests/suite/view_image.rs@@ -73,6 +73,13 @@ enum ResizeNoticeExpectation {Enabled,}+#[derive(Clone, Copy, Eq, PartialEq)]+enum ImageBudgetPolicy {+ DetailBased,+ Unified,+ UnifiedResponsesLiteWithoutOriginalSupport,+}+fn disabled_user_turn(test: &TestCodex, items: Vec<UserInput>, model: String) -> Op {let (sandbox_policy, permission_profile) =turn_permission_fields(PermissionProfile::Disabled, test.config.cwd.as_path());@@ -204,17 +211,27 @@ async fn write_workspace_png(async fn assert_user_turn_local_image_resizes_to(original_dimensions: (u32, u32),expected_dimensions: (u32, u32),+ image_budget_policy: ImageBudgetPolicy,resize_notice_expectation: ResizeNoticeExpectation,) -> anyhow::Result<()> {let server = start_mock_server().await;- let builder = test_codex();- let mut builder = match resize_notice_expectation {- ResizeNoticeExpectation::Disabled => builder,- ResizeNoticeExpectation::Enabled => builder.with_config(|config| {- let _ = config.features.enable(Feature::ImageResizeNotice);- }),+ let builder = match image_budget_policy {+ ImageBudgetPolicy::DetailBased | ImageBudgetPolicy::Unified => test_codex(),+ ImageBudgetPolicy::UnifiedResponsesLiteWithoutOriginalSupport => test_codex()+ .with_model_info_override("gpt-5.4", |model_info| {+ model_info.supports_image_detail_original = false;+ model_info.use_responses_lite = true;+ }),};+ let mut builder = builder.with_config(move |config| {+ if image_budget_policy != ImageBudgetPolicy::DetailBased {+ let _ = config.features.enable(Feature::UnifiedImageBudget);+ }+ if matches!(resize_notice_expectation, ResizeNoticeExpectation::Enabled) {+ let _ = config.features.enable(Feature::ImageResizeNotice);+ }+ });let test = builder.build_with_auto_env(&server).await?;let TestCodex {codex,@@ -329,6 +346,7 @@ async fn user_turn_with_local_image_attaches_image() -> anyhow::Result<()> {assert_user_turn_local_image_resizes_to((2304, 864),(2048, 768),+ ImageBudgetPolicy::DetailBased,ResizeNoticeExpectation::Disabled,).await@@ -341,6 +359,7 @@ async fn user_turn_with_vertical_local_image_resizes_to_square_bounds() -> anyhoassert_user_turn_local_image_resizes_to((1024, 4096),(512, 2048),+ ImageBudgetPolicy::DetailBased,ResizeNoticeExpectation::Disabled,).await@@ -353,11 +372,47 @@ async fn user_turn_local_image_applies_patch_budget_and_reports_resize() -> anyhassert_user_turn_local_image_resizes_to((2048, 2048),(1600, 1600),+ ImageBudgetPolicy::DetailBased,ResizeNoticeExpectation::Enabled,).await}+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]+async fn user_turn_unified_image_budget_enforces_dimension_and_patch_limits() -> anyhow::Result<()>+{+ skip_if_no_network!(Ok(()));++ for (source_dimensions, expected_dimensions, resize_notice_expectation) in [+ ((6401, 100), (6000, 94), ResizeNoticeExpectation::Disabled),+ ((3201, 3201), (3200, 3200), ResizeNoticeExpectation::Enabled),+ ] {+ assert_user_turn_local_image_resizes_to(+ source_dimensions,+ expected_dimensions,+ ImageBudgetPolicy::Unified,+ resize_notice_expectation,+ )+ .await?;+ }++ Ok(())+}++#[tokio::test(flavor = "multi_thread", worker_threads = 2)]+async fn user_turn_unified_image_budget_supports_responses_lite_without_original_detail()+-> anyhow::Result<()> {+ skip_if_no_network!(Ok(()));++ assert_user_turn_local_image_resizes_to(+ (2304, 864),+ (2304, 864),+ ImageBudgetPolicy::UnifiedResponsesLiteWithoutOriginalSupport,+ ResizeNoticeExpectation::Disabled,+ )+ .await+}+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]async fn view_image_tool_attaches_local_image() -> anyhow::Result<()> {skip_if_no_network!(Ok(()));@@ -869,6 +924,82 @@ async fn view_image_tool_can_preserve_original_resolution_when_requested_on_gpt5Ok(())}+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]+async fn view_image_unified_budget_hides_detail_but_accepts_legacy_hints() -> anyhow::Result<()> {+ skip_if_no_network!(Ok(()));++ let server = start_mock_server().await;+ let mut builder = test_codex().with_model("gpt-5.4").with_config(|config| {+ let _ = config.features.enable(Feature::UnifiedImageBudget);+ });+ let test = builder.build_with_auto_env(&server).await?;+ let rel_path = "assets/unified-example.png";+ write_workspace_png(+ &test,+ rel_path,+ /*width*/ 2304,+ /*height*/ 864,+ [0u8, 80, 255, 255],+ )+ .await?;++ let call_id = "view-image-unified";+ let first_mock = responses::mount_sse_once(+ &server,+ sse(vec![+ ev_response_created("resp-1"),+ ev_function_call(+ call_id,+ "view_image",+ &serde_json::json!({ "path": rel_path, "detail": "high" }).to_string(),+ ),+ ev_completed("resp-1"),+ ]),+ )+ .await;+ let second_mock = responses::mount_sse_once(+ &server,+ sse(vec![+ ev_assistant_message("msg-1", "done"),+ ev_completed("resp-2"),+ ]),+ )+ .await;++ test.submit_turn("show the screenshot").await?;++ let first_request = first_mock.single_request().body_json();+ let view_image_tool = first_request["tools"]+ .as_array()+ .and_then(|tools| tools.iter().find(|tool| tool["name"] == "view_image"))+ .context("view_image tool should be available")?;+ assert!(+ view_image_tool["parameters"]["properties"]+ .get("detail")+ .is_none(),+ "the unified image budget should not advertise detail"+ );++ let request = second_mock.single_request();+ let output = request.function_call_output(call_id);+ let output_items = output["output"]+ .as_array()+ .context("view_image should return image content")?;+ assert_eq!(output_items.len(), 1);+ assert_eq!(output_items[0]["detail"], "original");++ let image_url = output_items[0]["image_url"]+ .as_str()+ .context("view_image output should include image_url")?;+ let (_, payload) = image_url+ .split_once(',')+ .context("view_image image_url should include a base64 payload")?;+ let image = load_from_memory(&BASE64_STANDARD.decode(payload)?)?;+ assert_eq!(image.dimensions(), (2304, 864));++ Ok(())+}+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]async fn view_image_tool_errors_clearly_for_unsupported_detail_values() -> anyhow::Result<()> {skip_if_no_network!(Ok(()));@@ -1041,8 +1172,27 @@ async fn view_image_tool_treats_null_detail_as_omitted() -> anyhow::Result<()> {async fn view_image_tool_resizes_when_model_lacks_original_detail_support() -> anyhow::Result<()> {skip_if_no_network!(Ok(()));+ assert_view_image_tool_resizes_without_original_support(ImageBudgetPolicy::DetailBased).await+}++#[tokio::test(flavor = "multi_thread", worker_threads = 2)]+async fn view_image_unified_budget_stays_disabled_for_unsupported_model() -> anyhow::Result<()> {+ skip_if_no_network!(Ok(()));++ assert_view_image_tool_resizes_without_original_support(ImageBudgetPolicy::Unified).await+}++async fn assert_view_image_tool_resizes_without_original_support(+ image_budget_policy: ImageBudgetPolicy,+) -> anyhow::Result<()> {let server = start_mock_server().await;- let mut builder = test_codex().with_model("gpt-5.2");+ let mut builder = test_codex()+ .with_model("gpt-5.2")+ .with_config(move |config| {+ if image_budget_policy == ImageBudgetPolicy::Unified {+ let _ = config.features.enable(Feature::UnifiedImageBudget);+ }+ });let test = builder.build_with_auto_env(&server).await?;let TestCodex {codex,diff --git a/codex-rs/features/src/lib.rs b/codex-rs/features/src/lib.rsindex aaaa5b5a3a..74881e5637 100644--- a/codex-rs/features/src/lib.rs+++ b/codex-rs/features/src/lib.rs@@ -224,6 +224,8 @@ pub enum Feature {ImageGeneration,/// Tell the model when a prompt image was resized and include its dimensions.ImageResizeNotice,+ /// Apply one shared pixel and token budget to every image, regardless of legacy detail hints.+ UnifiedImageBudget,/// Removed compatibility flag for always-on centralized image preparation.ResizeAllImages,/// Removed compatibility flag for always-on response item IDs.@@ -1276,6 +1278,12 @@ pub const FEATURES: &[FeatureSpec] = &[stage: Stage::UnderDevelopment,default_enabled: false,},+ FeatureSpec {+ id: Feature::UnifiedImageBudget,+ key: "unified_image_budget",+ stage: Stage::UnderDevelopment,+ default_enabled: false,+ },FeatureSpec {id: Feature::ResizeAllImages,key: "resize_all_images",