Lightweight coding agent that runs in your terminal
Rust
Account for file images in context budgets and Guardian reviews (#46072)
Krish Chainanib974893c9004
diff --git a/codex-rs/core/src/compact_remote_v2_image_budget_tests.rs b/codex-rs/core/src/compact_remote_v2_image_budget_tests.rsindex 7c7d94ac8b..ea37694606 100644--- a/codex-rs/core/src/compact_remote_v2_image_budget_tests.rs+++ b/codex-rs/core/src/compact_remote_v2_image_budget_tests.rs@@ -25,6 +25,15 @@ fn image() -> ContentItem {}}+fn file_image() -> ContentItem {+ ContentItem::InputImage {+ image: ImageReference::File {+ file_id: "file_123".to_string(),+ },+ detail: Some(codex_protocol::models::ImageDetail::Original),+ }+}+fn text(value: &str) -> ContentItem {ContentItem::InputText {text: value.to_string(),@@ -45,21 +54,23 @@ fn trim(items: Vec<ResponseItem>, max_tokens: usize) -> Vec<ResponseItem> {#[test]fn image_only_boundary_is_atomic_and_does_not_backfill_older_messages() {let newest = message(vec![text("new")]);- let items = vec![- message(vec![text("old")]),- message(vec![image()]),- newest.clone(),- ];- let image_tokens = images::content_item_token_count(&image());- for (max_tokens, expected) in [- (- image_tokens + 1,- vec![message(vec![image()]), newest.clone()],- ),- (image_tokens, vec![newest.clone()]),- (1, vec![newest]),- ] {- assert_eq!(trim(items.clone(), max_tokens), expected);+ for image in [image(), file_image()] {+ let items = vec![+ message(vec![text("old")]),+ message(vec![image.clone()]),+ newest.clone(),+ ];+ let image_tokens = images::content_item_token_count(&image);+ for (max_tokens, expected) in [+ (+ image_tokens + 1,+ vec![message(vec![image.clone()]), newest.clone()],+ ),+ (image_tokens, vec![newest.clone()]),+ (1, vec![newest.clone()]),+ ] {+ assert_eq!(trim(items.clone(), max_tokens), expected);+ }}}diff --git a/codex-rs/core/src/compact_remote_v2_images.rs b/codex-rs/core/src/compact_remote_v2_images.rsindex 94cbdc239e..6e4914e9a4 100644--- a/codex-rs/core/src/compact_remote_v2_images.rs+++ b/codex-rs/core/src/compact_remote_v2_images.rs@@ -1,10 +1,9 @@-use crate::context_manager::estimate_image_bytes;+use crate::context_manager::estimate_image_reference_bytes;use codex_context_fragments::AnnotatedContent;use codex_context_fragments::set_annotated_content;use codex_context_fragments::to_annotated_content;use codex_history::ResponseItemEnvelope;use codex_protocol::models::ContentItem;-use codex_protocol::models::ImageReference;use codex_protocol::models::is_image_close_tag_text;use codex_protocol::models::is_image_open_tag_text;use codex_protocol::models::is_local_image_open_tag_text;@@ -18,17 +17,10 @@ pub(super) fn content_item_token_count(item: &ContentItem) -> usize {ContentItem::InputText { text } | ContentItem::OutputText { text } => {approx_token_count(text)}- ContentItem::InputImage {- image: ImageReference::Inline { image_url },- detail,- } => usize::try_from(approx_tokens_from_byte_count_i64(estimate_image_bytes(- image_url, *detail,- )))+ ContentItem::InputImage { image, detail } => usize::try_from(+ approx_tokens_from_byte_count_i64(estimate_image_reference_bytes(image, *detail)),+ ).unwrap_or(usize::MAX),- ContentItem::InputImage {- image: ImageReference::File { .. },- ..- } => 0,ContentItem::InputAudio { .. } => 0,}}diff --git a/codex-rs/core/src/context_manager/history.rs b/codex-rs/core/src/context_manager/history.rsindex 883731f67c..b7ec2e4802 100644--- a/codex-rs/core/src/context_manager/history.rs+++ b/codex-rs/core/src/context_manager/history.rs@@ -890,6 +890,7 @@ fn estimate_encrypted_function_output_length(encoded_len: usize) -> usize {/// Returns the same coarse, model-visible token estimate used for full history estimates.////// Counts content directly, excluding transport IDs, metadata, and outer JSON escaping.+/// Original-detail file images use the maximum patch count.pub(crate) fn estimate_item_token_count(item: &ResponseItem) -> i64 {let model_visible_bytes = estimate_response_item_model_visible_bytes(item);approx_tokens_from_byte_count_i64(model_visible_bytes)@@ -918,7 +919,6 @@ static ORIGINAL_IMAGE_ESTIMATE_CACHE: LazyLock<BlockingLruCache<[u8; 20], Option});fn estimate_response_item_model_visible_bytes(item: &ResponseItem) -> i64 {- // TODO(kc) Account for file-backed image size after its token-cost contract is defined.match item {ResponseItem::Message { content, .. } => content.iter()@@ -926,14 +926,9 @@ fn estimate_response_item_model_visible_bytes(item: &ResponseItem) -> i64 {ContentItem::InputText { text } | ContentItem::OutputText { text } => {text_bytes(text)}- ContentItem::InputImage {- image: ImageReference::Inline { image_url },- detail,- } => estimate_image_bytes(image_url, *detail),- ContentItem::InputImage {- image: ImageReference::File { .. },- ..- } => 0,+ ContentItem::InputImage { image, detail } => {+ estimate_image_reference_bytes(image, *detail)+ }ContentItem::InputAudio { audio_url } => estimate_audio_bytes(audio_url),}).fold(0i64, i64::saturating_add),@@ -1112,8 +1107,8 @@ fn estimate_original_image_bytes(image_url: &str) -> Option<i64> {})}-/// Shared image estimate, excluding message framing.-pub(crate) fn estimate_image_bytes(image_url: &str, detail: Option<ImageDetail>) -> i64 {+/// Inline image estimate, excluding the data URL prefix and message framing.+fn estimate_image_bytes(image_url: &str, detail: Option<ImageDetail>) -> i64 {match detail {Some(ImageDetail::Original) => {estimate_original_image_bytes(image_url).unwrap_or(RESIZED_IMAGE_BYTES_ESTIMATE)@@ -1122,6 +1117,21 @@ pub(crate) fn estimate_image_bytes(image_url: &str, detail: Option<ImageDetail>)}}+/// Image estimate for callers that only have the reference. Original-detail file images use the+/// maximum patch count because their dimensions are not available from the reference.+pub(crate) fn estimate_image_reference_bytes(+ image: &ImageReference,+ detail: Option<ImageDetail>,+) -> i64 {+ match image {+ ImageReference::Inline { image_url } => estimate_image_bytes(image_url, detail),+ ImageReference::File { .. } if detail == Some(ImageDetail::Original) => {+ i64::try_from(approx_bytes_for_tokens(ORIGINAL_IMAGE_MAX_PATCHES)).unwrap_or(i64::MAX)+ }+ ImageReference::File { .. } => RESIZED_IMAGE_BYTES_ESTIMATE,+ }+}+fn estimate_audio_bytes(audio_url: &str) -> i64 {i64::try_from(approx_bytes_for_tokens(estimate_audio_token_count(audio_url,@@ -1136,14 +1146,9 @@ fn estimate_function_output_bytes(output: &FunctionCallOutputBody) -> i64 {.iter().map(|part| match part {FunctionCallOutputContentItem::InputText { text } => text_bytes(text),- FunctionCallOutputContentItem::InputImage {- image: ImageReference::Inline { image_url },- detail,- } => estimate_image_bytes(image_url, *detail),- FunctionCallOutputContentItem::InputImage {- image: ImageReference::File { .. },- ..- } => 0,+ FunctionCallOutputContentItem::InputImage { image, detail } => {+ estimate_image_reference_bytes(image, *detail)+ }FunctionCallOutputContentItem::InputAudio { audio_url } => {estimate_audio_bytes(audio_url)}diff --git a/codex-rs/core/src/context_manager/history_tests.rs b/codex-rs/core/src/context_manager/history_tests.rsindex a145ee20fb..704b8a56cf 100644--- a/codex-rs/core/src/context_manager/history_tests.rs+++ b/codex-rs/core/src/context_manager/history_tests.rs@@ -2577,8 +2577,39 @@ fn image_data_url_payload_does_not_dominate_message_estimate() {assert!(estimated > text_only_estimated);}+/// File images use the fixed estimate for normal detail and the maximum patch count for original.#[test]-fn image_data_url_payload_does_not_dominate_function_call_output_estimate() {+fn file_images_use_detail_appropriate_estimates() {+ let item = ResponseItem::Message {+ id: None,+ role: "user".to_string(),+ content: vec![+ ContentItem::InputImage {+ image: ImageReference::File {+ file_id: "file_high".to_string(),+ },+ detail: Some(ImageDetail::High),+ },+ ContentItem::InputImage {+ image: ImageReference::File {+ file_id: "file_original".to_string(),+ },+ detail: Some(ImageDetail::Original),+ },+ ],+ phase: None,+ internal_chat_message_metadata_passthrough: None,+ };++ let estimated = estimate_response_item_model_visible_bytes(&item);+ let expected = RESIZED_IMAGE_BYTES_ESTIMATE+ .saturating_add(approx_bytes_for_tokens(/*tokens*/ 10_000) as i64);++ assert_eq!(estimated, expected);+}++#[test]+fn function_call_output_estimates_inline_and_file_images() {let payload = "B".repeat(50_000);let image_url = format!("data:image/png;base64,{payload}");let item = ResponseItem::FunctionCallOutput {@@ -2594,14 +2625,22 @@ fn image_data_url_payload_does_not_dominate_function_call_output_estimate() {image: ImageReference::Inline { image_url },detail: Some(DEFAULT_IMAGE_DETAIL),},+ FunctionCallOutputContentItem::InputImage {+ image: ImageReference::File {+ file_id: "file_original".to_string(),+ },+ detail: Some(ImageDetail::Original),+ },]),internal_chat_message_metadata_passthrough: None,};let raw_len = serde_json::to_string(&item).unwrap().len() as i64;let estimated = estimate_response_item_model_visible_bytes(&item);- let expected =- "call-abc".len() as i64 + "Screenshot captured".len() as i64 + RESIZED_IMAGE_BYTES_ESTIMATE;+ let expected = "call-abc".len() as i64+ + "Screenshot captured".len() as i64+ + RESIZED_IMAGE_BYTES_ESTIMATE+ + approx_bytes_for_tokens(/*tokens*/ 10_000) as i64;assert_eq!(estimated, expected);assert!(estimated < raw_len);diff --git a/codex-rs/core/src/context_manager/mod.rs b/codex-rs/core/src/context_manager/mod.rsindex af520df6cf..8f3a347d20 100644--- a/codex-rs/core/src/context_manager/mod.rs+++ b/codex-rs/core/src/context_manager/mod.rs@@ -4,6 +4,6 @@ pub(crate) mod updates;pub(crate) use history::ContextManager;pub(crate) use history::HistoryReplacement;-pub(crate) use history::estimate_image_bytes;+pub(crate) use history::estimate_image_reference_bytes;pub(crate) use history::estimate_item_token_count;pub(crate) use history::is_user_turn_boundary;diff --git a/codex-rs/core/src/guardian/review_session.rs b/codex-rs/core/src/guardian/review_session.rsindex 7179aa3175..d176191972 100644--- a/codex-rs/core/src/guardian/review_session.rs+++ b/codex-rs/core/src/guardian/review_session.rs@@ -508,20 +508,26 @@ async fn run_review_on_session(.any(|(_, cost)| cost.image_count > 0){let reviewer_history = review_session.session.clone_history().await;- let reviewer_image_urls = reviewer_history- .raw_items()- .flat_map(|item| match item {- ResponseItem::Message { content, .. } => content.as_slice(),- _ => &[],- })- .filter_map(|item| match item {- ContentItem::InputImage {- image: ImageReference::Inline { image_url },- ..- } => Some(image_url.as_str()),- _ => None,- })- .collect::<HashSet<_>>();+ let mut reviewer_image_urls = HashSet::new();+ let mut reviewer_file_ids = HashSet::new();+ for item in reviewer_history.raw_items() {+ let ResponseItem::Message { content, .. } = item else {+ continue;+ };+ for item in content {+ let ContentItem::InputImage { image, .. } = item else {+ continue;+ };+ match image {+ ImageReference::Inline { image_url } => {+ reviewer_image_urls.insert(image_url.as_str());+ }+ ImageReference::File { file_id } => {+ reviewer_file_ids.insert(file_id.as_str());+ }+ }+ }+ }let context_window = model_info.resolved_context_window().map(|supported| {params.spawn_config@@ -542,7 +548,7 @@ async fn run_review_on_session(} else {ImagePreparationMode::DetailBased};- prompt_items.context.retain_images(|image_url, detail| {+ prompt_items.context.retain_images(|image, detail| {*detail = match normalize_output_image_detail(&model_info, *detail) {_ if mode == ImagePreparationMode::UnifiedBudget => {Some(ImageDetail::Original)@@ -550,15 +556,23 @@ async fn run_review_on_session(Some(ImageDetail::Low) => Some(ImageDetail::High),detail => detail,};- let prepared_image_url = match resize_image(image_url, detail, mode) {- Ok(Some(prepared)) => Cow::Owned(prepared.into_data_url()),- Ok(None) => Cow::Borrowed(image_url),- Err(error) => {- warn!(%error, "failed to prepare guardian review image");- return false;+ match image {+ ImageReference::Inline { image_url } => {+ let prepared_image_url = match resize_image(image_url, detail, mode)+ {+ Ok(Some(prepared)) => Cow::Owned(prepared.into_data_url()),+ Ok(None) => Cow::Borrowed(image_url),+ Err(error) => {+ warn!(%error, "failed to prepare guardian review image");+ return false;+ }+ };+ !reviewer_image_urls.contains(prepared_image_url.as_str())}- };- !reviewer_image_urls.contains(prepared_image_url.as_ref())+ ImageReference::File { file_id } => {+ !reviewer_file_ids.contains(file_id.as_str())+ }+ }});let prompt: ResponseItem =ResponseInputItem::from(prompt_items.context.clone().into_user_inputs()?)diff --git a/codex-rs/core/tests/suite/compact_remote.rs b/codex-rs/core/tests/suite/compact_remote.rsindex 82ed5f5092..f7dc16e13d 100644--- a/codex-rs/core/tests/suite/compact_remote.rs+++ b/codex-rs/core/tests/suite/compact_remote.rs@@ -605,11 +605,14 @@ async fn amazon_bedrock_automatic_compaction_uses_v2_responses_endpoint() -> ResOk(())}-#[test_case(None; "default_trims_images")]-#[test_case(Some(false); "disabled_preserves_images")]+#[test_case(None, "image_url"; "default_trims_images")]+#[test_case(Some(false), "image_url"; "disabled_preserves_images")]+#[test_case(None, "file_id"; "default_trims_file_images")]+#[test_case(Some(false), "file_id"; "disabled_preserves_file_images")]#[tokio::test(flavor = "multi_thread", worker_threads = 2)]async fn remote_compact_v2_charges_retained_images_to_token_budget(image_budget_enabled: Option<bool>,+ image_field: &str,) -> Result<()> {skip_if_no_network!(Ok(()));@@ -630,6 +633,14 @@ async fn remote_compact_v2_charges_retained_images_to_token_budget(// Each original-detail image costs 10,000 estimated patch tokens.let image_inputs = (1..=8).map(|number| {+ if image_field == "file_id" {+ return Ok(UserInput::Image {+ image: ImageReference::File {+ file_id: format!("file_{number}"),+ },+ detail: Some(codex_protocol::models::ImageDetail::Original),+ });+ }let image = image::ImageBuffer::from_pixel(/*width*/ 3200,/*height*/ 3200,@@ -666,7 +677,24 @@ async fn remote_compact_v2_charges_retained_images_to_token_budget(.await?;wait_for_turn_complete(codex).await;let initial_request = initial_mock.single_request();- let prepared_images = initial_request.message_input_image_urls("user");+ let prepared_images = initial_request+ .inputs_of_type("message")+ .into_iter()+ .filter(|item| item["role"] == "user")+ .flat_map(|item| {+ item["content"]+ .as_array()+ .expect("message content is an array")+ .clone()+ })+ .filter(|item| item["type"] == "input_image")+ .map(|item| {+ item[image_field]+ .as_str()+ .expect("image input has a string reference")+ .to_owned()+ })+ .collect::<Vec<_>>();assert_eq!(prepared_images.len(), 7);for cycle in 1..=2 {@@ -717,16 +745,33 @@ async fn remote_compact_v2_charges_retained_images_to_token_budget(};let mut expected_images = prepared_images[dropped..].to_vec();if cycle == 2 {- let UserInput::Image {- image: ImageReference::Inline { image_url },- ..- } = &image_inputs[7]- else {+ let UserInput::Image { image, .. } = &image_inputs[7] else {unreachable!()};- expected_images.push(image_url.clone());+ expected_images.push(match image {+ ImageReference::Inline { image_url } => image_url.clone(),+ ImageReference::File { file_id } => file_id.clone(),+ });}- assert_eq!(follow_up.message_input_image_urls("user"), expected_images);+ let retained_images = follow_up+ .inputs_of_type("message")+ .into_iter()+ .filter(|item| item["role"] == "user")+ .flat_map(|item| {+ item["content"]+ .as_array()+ .expect("message content is an array")+ .clone()+ })+ .filter(|item| item["type"] == "input_image")+ .map(|item| {+ item[image_field]+ .as_str()+ .expect("image input has a string reference")+ .to_owned()+ })+ .collect::<Vec<_>>();+ assert_eq!(retained_images, expected_images);assert!(follow_up.message_input_texts("user")diff --git a/codex-rs/core/tests/suite/guardian_context_budget.rs b/codex-rs/core/tests/suite/guardian_context_budget.rsindex 29be5e7a59..b2c658d730 100644--- a/codex-rs/core/tests/suite/guardian_context_budget.rs+++ b/codex-rs/core/tests/suite/guardian_context_budget.rs@@ -30,8 +30,11 @@ use core_test_support::test_codex::test_codex;use core_test_support::wait_for_event;use pretty_assertions::assert_eq;use serde_json::json;+use std::sync::Arc;use test_case::test_case;+use super::image_rollout::RecordingFileAttachmentStore;+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]#[test_case(160_000, &[0, 0]; "complete_instructions_fit")]#[test_case(28_000, &[0, 1]; "oversized_history_compacts_then_truncates")]@@ -198,6 +201,7 @@ async fn review_preserves_user_instructions_until_request_budgeting(enum ReviewerResponse {Decision,ToolContinuation,+ FileImageContinuation,UncompactableContinuation,CompactionError,NextReview,@@ -206,6 +210,7 @@ enum ReviewerResponse {#[tokio::test(flavor = "multi_thread", worker_threads = 2)]#[test_case(1, ReviewerResponse::Decision; "required_context_fails_closed")]#[test_case(4_500, ReviewerResponse::ToolContinuation; "oversized_tool_continuation_compacts")]+#[test_case(4_500, ReviewerResponse::FileImageContinuation; "uploaded_original_image_history_compacts")]#[test_case(4_500, ReviewerResponse::UncompactableContinuation; "ineffective_compaction_fails_closed")]#[test_case(4_500, ReviewerResponse::CompactionError; "compaction_service_error_does_not_request_user_approval")]#[test_case(6_000, ReviewerResponse::NextReview; "incoming_review_compacts_existing_history")]@@ -231,6 +236,10 @@ async fn review_respects_complete_context_budget().with_model_info_override("gpt-5.6-luna", move |model| {model.context_window = Some(window);+ if matches!(reviewer_response, ReviewerResponse::FileImageContinuation) {+ model.supports_image_detail_original = true;+ model.use_responses_lite = false;+ }model.model_messages.as_mut()@@ -268,12 +277,17 @@ async fn review_respects_complete_context_budget(if matches!(reviewer_response,ReviewerResponse::ToolContinuation+ | ReviewerResponse::FileImageContinuation| ReviewerResponse::UncompactableContinuation| ReviewerResponse::CompactionError) {builder = builder.with_code_mode_host_program(codex_utils_cargo_bin::cargo_bin("codex-code-mode-host")?);}+ let image_store = Arc::new(RecordingFileAttachmentStore::default());+ if matches!(reviewer_response, ReviewerResponse::FileImageContinuation) {+ builder = builder.with_image_store(image_store.clone());+ }let test = builder.build_with_auto_env(&server).await?;let command = json!({"cmd": "echo guardian-budget-test", "sandbox_permissions": "require_escalated", "justification": "Run the requested command."}).to_string();let mut commentary =@@ -321,12 +335,22 @@ async fn review_respects_complete_context_budget("exec","text('inspection-output'.repeat(600));",),+ // A tiny inline image fits before upload. Its opaque original-detail file+ // reference must reserve 10k tokens in reviewer history and force compaction.+ ReviewerResponse::FileImageContinuation => ev_custom_tool_call(+ "reviewer-inspect",+ "exec",+ r#"image("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGP4z8DwHwAFAAH/iZk9HQAAAABJRU5ErkJggg==", "original");"#,+ ),},ev_completed("review"),]),);}- if matches!(reviewer_response, ReviewerResponse::ToolContinuation) {+ if matches!(+ reviewer_response,+ ReviewerResponse::ToolContinuation | ReviewerResponse::FileImageContinuation+ ) {events.insert(/*index*/ 2,sse(vec![@@ -424,18 +448,38 @@ async fn review_respects_complete_context_budget("expected required-evidence budget rejection: {output}");} else {- let recovered = matches!(reviewer_response, ReviewerResponse::ToolContinuation);+ let recovered = matches!(+ reviewer_response,+ ReviewerResponse::ToolContinuation | ReviewerResponse::FileImageContinuation+ );assert_eq!(requests.len(), if recovered { 4 } else { 3 });assert_eq!(guardian_requests.len(), if recovered { 2 } else { 1 });if recovered {assert_eq!(compact_requests.len(), 1);let compact = &compact_requests[0];- assert!(- compact- .body_json()- .to_string()- .contains("inspection-output")- );+ if matches!(reviewer_response, ReviewerResponse::FileImageContinuation) {+ assert_eq!(+ image_store+ .uploads+ .lock()+ .expect("image upload tracker lock is not poisoned")+ .len(),+ 1+ );+ // The file reservation also protects the compaction request itself: an output+ // larger than its window is replaced before the summary request is sent.+ assert_eq!(+ compact.custom_tool_call_output("reviewer-inspect")["output"],+ "Output exceeded the available model context and was truncated"+ );+ } else {+ assert!(+ compact+ .body_json()+ .to_string()+ .contains("inspection-output")+ );+ }let recovered = guardian_requests[1].body_json();assert!(recovered["input"]diff --git a/codex-rs/core/tests/suite/image_rollout.rs b/codex-rs/core/tests/suite/image_rollout.rsindex b30ad11aa9..fb749dc813 100644--- a/codex-rs/core/tests/suite/image_rollout.rs+++ b/codex-rs/core/tests/suite/image_rollout.rs@@ -57,8 +57,8 @@ use std::time::Duration;const UPLOADED_FILE_ID: &str = "file_uploaded_image";#[derive(Default)]-struct RecordingFileAttachmentStore {- uploads: Mutex<Vec<UploadRequest>>,+pub(super) struct RecordingFileAttachmentStore {+ pub(super) uploads: Mutex<Vec<UploadRequest>>,}impl AttachmentStore for RecordingFileAttachmentStore {diff --git a/codex-rs/ext/guardian-v2/src/async_scorer/extension_tests.rs b/codex-rs/ext/guardian-v2/src/async_scorer/extension_tests.rsindex 2f4c531b29..c2454b8008 100644--- a/codex-rs/ext/guardian-v2/src/async_scorer/extension_tests.rs+++ b/codex-rs/ext/guardian-v2/src/async_scorer/extension_tests.rs@@ -1677,6 +1677,8 @@ async fn contributor_includes_transcript_images_by_default() -> Result<()> {let user_image = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVR4nGPgEpEDAABoAD1UCKP3AAAAAElFTkSuQmCC";let tool_image = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVR4nGOQE+ECAACQAD304kFaAAAAAElFTkSuQmCC";+ let user_file_id = "file_user";+ let tool_file_id = "file_tool";let history = vec![ResponseItem::Message {id: None,@@ -1691,6 +1693,12 @@ async fn contributor_includes_transcript_images_by_default() -> Result<()> {},detail: Some(ImageDetail::High),},+ ContentItem::InputImage {+ image: ImageReference::File {+ file_id: user_file_id.to_owned(),+ },+ detail: Some(ImageDetail::High),+ },],phase: None,internal_chat_message_metadata_passthrough: None,@@ -1719,6 +1727,12 @@ async fn contributor_includes_transcript_images_by_default() -> Result<()> {},detail: Some(ImageDetail::High),},+ FunctionCallOutputContentItem::InputImage {+ image: ImageReference::File {+ file_id: tool_file_id.to_owned(),+ },+ detail: Some(ImageDetail::High),+ },]),internal_chat_message_metadata_passthrough: None,},@@ -1740,16 +1754,24 @@ enabled = true.expect("Luna user content should be an array");assert_eq!(- content[content.len() - 2..],+ content[content.len() - 4..],[json!({"type": "input_image","image_url": user_image,}),+ json!({+ "type": "input_image",+ "file_id": user_file_id,+ }),json!({"type": "input_image","image_url": tool_image,}),+ json!({+ "type": "input_image",+ "file_id": tool_file_id,+ }),]);diff --git a/codex-rs/guardian-context/src/budget.rs b/codex-rs/guardian-context/src/budget.rsindex 1ebec846e4..06db2a906d 100644--- a/codex-rs/guardian-context/src/budget.rs+++ b/codex-rs/guardian-context/src/budget.rs@@ -108,7 +108,9 @@ impl SectionCost {ContentItem::InputImage {image: ImageReference::File { .. },..- } => {}+ } => {+ self.image_count = self.image_count.saturating_add(1);+ }ContentItem::InputAudio { audio_url } => {// Guardian currently has no audio contributor. Count a future opaque// payload conservatively until its consumer supplies modality costs.@@ -214,17 +216,16 @@ pub(super) fn content_framing_tokens(item_count: usize) -> usize {fn adjusted_tokens(mut bytes: usize, content: &[ContentItem]) -> usize {for item in content {- if let ContentItem::InputImage {- image: ImageReference::Inline { image_url },- ..- } = item- {- let payload = ByteCount::measure(|counter| serde_json::to_writer(counter, image_url));- if payload == usize::MAX {- return usize::MAX;+ if let ContentItem::InputImage { image, .. } = item {+ if let ImageReference::Inline { image_url } = image {+ let payload =+ ByteCount::measure(|counter| serde_json::to_writer(counter, image_url));+ if payload == usize::MAX {+ return usize::MAX;+ }+ bytes = bytes.saturating_sub(payload.saturating_sub(2));}bytes = bytes- .saturating_sub(payload.saturating_sub(2)).saturating_add(TruncationPolicy::Tokens(IMAGE_TOKEN_RESERVATION).byte_budget());}}diff --git a/codex-rs/guardian-context/src/budget_tests.rs b/codex-rs/guardian-context/src/budget_tests.rsindex 33fc174ad7..8ce4fbb4a6 100644--- a/codex-rs/guardian-context/src/budget_tests.rs+++ b/codex-rs/guardian-context/src/budget_tests.rs@@ -23,6 +23,12 @@ fn section_costs_keep_multimodal_payloads_separate() {},detail: None,}),+ Budgeted::required(ContentItem::InputImage {+ image: ImageReference::File {+ file_id: "file_123".to_owned(),+ },+ detail: None,+ }),]),},SectionOutput {@@ -44,7 +50,7 @@ fn section_costs_keep_multimodal_payloads_separate() {SectionCost {text_bytes: "évidence".len(),image_bytes: "data:image/png;base64,AAAA".len(),- image_count: 1+ image_count: 2}),(@@ -71,6 +77,14 @@ fn request_estimate_reserves_images_independently_of_encoded_size() {let short = estimate_input_tokens(&image("AAAA"));assert_eq!(short, estimate_input_tokens(&image(&"A".repeat(200_000))));assert!(short >= IMAGE_TOKEN_RESERVATION);++ let file = message(vec![ContentItem::InputImage {+ image: ImageReference::File {+ file_id: "file_123".to_owned(),+ },+ detail: Some(codex_protocol::models::ImageDetail::Original),+ }]);+ assert!(estimate_input_tokens(&file) >= IMAGE_TOKEN_RESERVATION);}#[test]diff --git a/codex-rs/guardian-context/src/composition.rs b/codex-rs/guardian-context/src/composition.rsindex 66892875fd..25450d989d 100644--- a/codex-rs/guardian-context/src/composition.rs+++ b/codex-rs/guardian-context/src/composition.rs@@ -292,17 +292,7 @@ impl ComposedContext {}));continue;}- ContentItem::InputImage {- image: ImageReference::Inline { image_url },- detail,- } => UserInput::Image {- image: ImageReference::Inline { image_url },- detail,- },- ContentItem::InputImage {- image: ImageReference::File { .. },- ..- } => continue,+ ContentItem::InputImage { image, detail } => UserInput::Image { image, detail },ContentItem::InputAudio { .. } | ContentItem::OutputText { .. } => {return Err(SectionError::UnsupportedDelivery {section: section.id,@@ -330,10 +320,6 @@ impl ComposedContext {}}));}- ContentItem::InputImage {- image: ImageReference::File { .. },- ..- } => {}content => user_content.push(content),}}diff --git a/codex-rs/guardian-context/src/composition_tests.rs b/codex-rs/guardian-context/src/composition_tests.rsindex 4095f00748..ed0fdf41c1 100644--- a/codex-rs/guardian-context/src/composition_tests.rs+++ b/codex-rs/guardian-context/src/composition_tests.rs@@ -1,4 +1,6 @@use super::*;+use codex_protocol::models::ImageDetail;+use codex_protocol::models::ImageReference;use pretty_assertions::assert_eq;#[test]@@ -56,12 +58,21 @@ fn delivery_preserves_arbitrary_message_boundaries_and_rejects_them_for_sync() {}#[test]-fn long_text_delivery_is_lossless_bounded_and_fully_budgeted() {+fn long_text_and_file_image_delivery_is_lossless_bounded_and_fully_budgeted() {let text = "é🙂\"\n".repeat(/*n*/ 20_000);+ let file_image = ImageReference::File {+ file_id: "file_123".to_owned(),+ };let context = ComposedContext {sections: vec![SectionOutput {id: "planned_action",- delivery: text_content(vec![text.clone()]),+ delivery: SectionDelivery::UserContent(vec![+ Budgeted::required(ContentItem::InputText { text: text.clone() }),+ Budgeted::required(ContentItem::InputImage {+ image: file_image.clone(),+ detail: Some(ImageDetail::High),+ }),+ ]),}],truncations: Vec::new(),};@@ -70,7 +81,15 @@ fn long_text_delivery_is_lossless_bounded_and_fully_budgeted() {let ResponseItem::Message { content, .. } = &messages[0] else {panic!("user message")};- let parts = content+ let (image, text_content) = content.split_last().expect("message content");+ assert_eq!(+ image,+ &ContentItem::InputImage {+ image: file_image.clone(),+ detail: Some(ImageDetail::High),+ }+ );+ let parts = text_content.iter().map(|item| match item {ContentItem::InputText { text } => text.as_str(),@@ -84,14 +103,16 @@ fn long_text_delivery_is_lossless_bounded_and_fully_budgeted() {.all(|part| part.len() <= TruncationPolicy::Tokens(9_000).byte_budget()));assert!(estimated >= crate::estimate_input_tokens(&messages[0]));- assert_eq!(- context.into_user_inputs().unwrap(),- parts- .into_iter()- .map(|part| UserInput::Text {- text: part.to_owned(),- text_elements: Vec::new(),- })- .collect::<Vec<_>>()- );+ let mut expected_inputs = parts+ .into_iter()+ .map(|part| UserInput::Text {+ text: part.to_owned(),+ text_elements: Vec::new(),+ })+ .collect::<Vec<_>>();+ expected_inputs.push(UserInput::Image {+ image: file_image,+ detail: Some(ImageDetail::High),+ });+ assert_eq!(context.into_user_inputs().unwrap(), expected_inputs);}diff --git a/codex-rs/guardian-context/src/enforcement.rs b/codex-rs/guardian-context/src/enforcement.rsindex 4e59d73463..234f0d2fbd 100644--- a/codex-rs/guardian-context/src/enforcement.rs+++ b/codex-rs/guardian-context/src/enforcement.rs@@ -88,17 +88,13 @@ pub enum HistoryTruncation {impl ComposedContext {/// Applies host image admission before aggregate selection, preserving section/// identity and each retained item's selection policy.- pub fn retain_images(&mut self, mut admit: impl FnMut(&str, &mut Option<ImageDetail>) -> bool) {+ pub fn retain_images(+ &mut self,+ mut admit: impl FnMut(&ImageReference, &mut Option<ImageDetail>) -> bool,+ ) {for section in &mut self.sections {retain_content(section, &mut self.truncations, |_, item| match item {- ContentItem::InputImage {- image: ImageReference::Inline { image_url },- detail,- } => admit(image_url, detail),- ContentItem::InputImage {- image: ImageReference::File { .. },- ..- } => false,+ ContentItem::InputImage { image, detail } => admit(image, detail),_ => true,});}diff --git a/codex-rs/guardian-context/src/enforcement_tests.rs b/codex-rs/guardian-context/src/enforcement_tests.rsindex 88599eb177..4c3045753b 100644--- a/codex-rs/guardian-context/src/enforcement_tests.rs+++ b/codex-rs/guardian-context/src/enforcement_tests.rs@@ -236,8 +236,14 @@ fn budget_reserves_existing_context_and_preserves_required_messages() {}#[test]-fn image_omission_preserves_text_and_later_eviction_policy() {+fn image_accounting_preserves_later_eviction_policy() {let evidence = text(&"optional commentary ".repeat(/*n*/ 100));+ let file_image = ContentItem::InputImage {+ image: ImageReference::File {+ file_id: "file_123".to_owned(),+ },+ detail: None,+ };let mut context = ComposedContext {sections: vec![SectionOutput {id: "evidence",@@ -251,6 +257,7 @@ fn image_omission_preserves_text_and_later_eviction_policy() {},BudgetPriority::Image,),+ Budgeted::optional(file_image.clone(), BudgetPriority::Image),Budgeted::optional(evidence.clone(), BudgetPriority::Commentary),Budgeted::required(text("user restriction")),]),@@ -261,7 +268,7 @@ fn image_omission_preserves_text_and_later_eviction_policy() {.clone().enforce_budget(RequestBudget {- max_input_tokens: 1_000,+ max_input_tokens: content_tokens(&file_image).saturating_add(1_000),existing_context_tokens: 0,},"evidence omitted".to_owned(),@@ -271,12 +278,12 @@ fn image_omission_preserves_text_and_later_eviction_policy() {assert_eq!(without_oversized_image.into_messages(),vec![user_message(vec![- evidence.clone(),+ file_image.clone(),text("user restriction"),text("evidence omitted")])]);- context.retain_images(|_, _| false);+ context.retain_images(|image, _| matches!(image, ImageReference::File { .. }));let available = context.estimated_tokens();let retained = context.clone()@@ -291,12 +298,16 @@ fn image_omission_preserves_text_and_later_eviction_policy() {.unwrap();assert_eq!(retained.into_messages(),- vec![user_message(vec![evidence, text("user restriction")])]+ vec![user_message(vec![+ file_image.clone(),+ evidence,+ text("user restriction")+ ])]);let smaller = context.enforce_budget(RequestBudget {- max_input_tokens: 100,+ max_input_tokens: content_tokens(&file_image).saturating_add(100),existing_context_tokens: 0,},"evidence omitted".to_owned(),@@ -306,6 +317,7 @@ fn image_omission_preserves_text_and_later_eviction_policy() {assert_eq!(smaller.into_messages(),vec![user_message(vec![+ file_image,text("user restriction"),text("evidence omitted")])]diff --git a/codex-rs/guardian-context/src/images.rs b/codex-rs/guardian-context/src/images.rsindex 27314df4db..507e688215 100644--- a/codex-rs/guardian-context/src/images.rs+++ b/codex-rs/guardian-context/src/images.rs@@ -1,5 +1,6 @@//! Bounded image selection shared by Guardian consumers.//! Keeps source order and evicts oldest images using the existing count/byte caps.+//! Byte caps cover inline URLs and file ID strings, not referenced file contents.//! Consumers still choose source visibility and model-specific image detail.use crate::ContextSection;@@ -29,6 +30,7 @@ pub struct TranscriptImageInput<'a> {#[derive(Clone, Default, PartialEq)]pub struct TranscriptImages {pub images: Vec<ContentItem>,+ /// Omitted inline payload or file ID bytes, not the size of referenced files.pub omitted_bytes: usize,}@@ -56,37 +58,37 @@ impl TranscriptImages {}let mut images = VecDeque::new();- let mut image_bytes = 0usize;+ let mut image_reference_bytes = 0usize;let mut omitted_bytes = 0usize;- let mut include_image = |image_url: &str, detail: Option<ImageDetail>| {- if image_url.len() > MAX_TRANSCRIPT_IMAGE_BYTES {- omitted_bytes = omitted_bytes.saturating_add(image_url.len());+ let mut include_image = |image: &ImageReference, detail: Option<ImageDetail>| {+ let reference_bytes = match image {+ ImageReference::Inline { image_url } => image_url.len(),+ ImageReference::File { file_id } => file_id.len(),+ };+ if reference_bytes > MAX_TRANSCRIPT_IMAGE_BYTES {+ omitted_bytes = omitted_bytes.saturating_add(reference_bytes);return;}while images.len() >= MAX_TRANSCRIPT_IMAGES- || image_bytes + image_url.len() > MAX_TRANSCRIPT_IMAGE_BYTES+ || image_reference_bytes + reference_bytes > MAX_TRANSCRIPT_IMAGE_BYTES{- let Some(ContentItem::InputImage {- image: ImageReference::Inline { image_url },- ..- }) = images.pop_front()- else {+ let Some(ContentItem::InputImage { image, .. }) = images.pop_front() else {break;};- image_bytes -= image_url.len();- omitted_bytes = omitted_bytes.saturating_add(image_url.len());+ let evicted_bytes = match image {+ ImageReference::Inline { image_url } => image_url.len(),+ ImageReference::File { file_id } => file_id.len(),+ };+ image_reference_bytes = image_reference_bytes.saturating_sub(evicted_bytes);+ omitted_bytes = omitted_bytes.saturating_add(evicted_bytes);}- image_bytes += image_url.len();+ image_reference_bytes = image_reference_bytes.saturating_add(reference_bytes);images.push_back(ContentItem::InputImage {- image: ImageReference::Inline {- image_url: image_url.to_owned(),- },+ image: image.clone(),detail,});};- // TODO(kc) Include file-backed images once Guardian can admit them without resolving or- // applying byte-based checks in Core.for item in items {match item {ResponseItem::Message { role, content, .. }@@ -94,15 +96,10 @@ impl TranscriptImages {{for item in content {match item {- ContentItem::InputImage {- image: ImageReference::Inline { image_url },- detail,- } => include_image(image_url, *detail),- ContentItem::InputImage {- image: ImageReference::File { .. },- ..+ ContentItem::InputImage { image, detail } => {+ include_image(image, *detail)}- | ContentItem::InputText { .. }+ ContentItem::InputText { .. }| ContentItem::InputAudio { .. }| ContentItem::OutputText { .. } => {}}@@ -115,15 +112,10 @@ impl TranscriptImages {if let Some(content) = output.content_items() {for item in content {match item {- FunctionCallOutputContentItem::InputImage {- image: ImageReference::Inline { image_url },- detail,- } => include_image(image_url, *detail),- FunctionCallOutputContentItem::InputImage {- image: ImageReference::File { .. },- ..+ FunctionCallOutputContentItem::InputImage { image, detail } => {+ include_image(image, *detail)}- | FunctionCallOutputContentItem::InputText { .. }+ FunctionCallOutputContentItem::InputText { .. }| FunctionCallOutputContentItem::InputAudio { .. }| FunctionCallOutputContentItem::EncryptedContent { .. } => {}}@@ -137,9 +129,9 @@ impl TranscriptImages {for image in input.node_repl_images {match image {ContentItem::InputImage {- image: ImageReference::Inline { image_url },+ image: image @ ImageReference::Inline { .. },detail,- } => include_image(image_url, *detail),+ } => include_image(image, *detail),ContentItem::InputImage {image: ImageReference::File { .. },..diff --git a/codex-rs/guardian-context/src/images_tests.rs b/codex-rs/guardian-context/src/images_tests.rsindex 5c3fa54d04..89361ffe83 100644--- a/codex-rs/guardian-context/src/images_tests.rs+++ b/codex-rs/guardian-context/src/images_tests.rs@@ -1,6 +1,11 @@use super::MAX_TRANSCRIPT_IMAGE_BYTES;use super::TranscriptImageInput;use super::TranscriptImages;+use crate::CollectedContext;+use crate::ContextPresentation;+use crate::ContextSection;+use crate::RenderedTranscript;+use crate::composition::user_message;use codex_protocol::models::ContentItem;use codex_protocol::models::FunctionCallOutputContentItem;use codex_protocol::models::FunctionCallOutputPayload;@@ -99,20 +104,26 @@ fn image_selection_preserves_source_policy_order_and_both_limits() {);}-/// File-backed images stay out of Guardian without consuming its inline-image capacity.+/// File-backed history images share count and reference-byte limits with inline images.#[test]-fn file_images_are_omitted_without_displacing_inline_images() {+fn file_images_are_selected_from_history_sources() {let file_image = ContentItem::InputImage {image: ImageReference::File {file_id: "file_123".to_string(),},detail: Some(ImageDetail::High),};+ let tool_file_image = ContentItem::InputImage {+ image: ImageReference::File {+ file_id: "file_tool".to_string(),+ },+ detail: Some(ImageDetail::High),+ };let history = [ResponseItem::Message {id: None,role: "user".into(),- content: vec![image("first"), file_image.clone(), image("second")],+ content: vec![image("first"), file_image.clone()],phase: None,internal_chat_message_metadata_passthrough: None,},@@ -138,29 +149,106 @@ fn file_images_are_omitted_without_displacing_inline_images() {internal_chat_message_metadata_passthrough: None,},];- let repl = [- file_image,- image("fourth"),- ContentItem::InputImage {+ let repl = [image("fourth")];++ assert_eq!(+ TranscriptImages::collect(+ &history,+ TranscriptImageInput {+ enabled: true,+ include_tool_outputs: true,+ node_repl_images: &repl,+ }+ ),+ TranscriptImages {+ images: vec![file_image, tool_file_image, image("third"), image("fourth")],+ omitted_bytes: "first".len(),+ }+ );++ // Count-only eviction must remain observable even when no inline bytes were omitted.+ let file_images =+ ["file_1", "file_2", "file_3", "file_4", "file_5"].map(|file_id| ContentItem::InputImage {image: ImageReference::File {- file_id: "file_repl".to_string(),+ file_id: file_id.to_owned(),},detail: Some(ImageDetail::High),+ });+ let history = [ResponseItem::Message {+ id: None,+ role: "user".into(),+ content: file_images.to_vec(),+ phase: None,+ internal_chat_message_metadata_passthrough: None,+ }];+ let selected = TranscriptImages::collect(+ &history,+ TranscriptImageInput {+ enabled: true,+ include_tool_outputs: true,+ node_repl_images: &[],},- ];+ );+ assert_eq!(+ selected,+ TranscriptImages {+ images: file_images[1..].to_vec(),+ omitted_bytes: "file_1".len(),+ }+ );+ let context = CollectedContext {+ sections: vec![ContextSection::TranscriptImages(selected)],+ }+ .compose(+ ContextPresentation::Async,+ RenderedTranscript {+ items: Vec::new(),+ omission_note: None,+ truncations: Vec::new(),+ },+ )+ .unwrap();+ assert_eq!(+ context+ .truncations+ .iter()+ .map(|observation| (+ observation.component,+ observation.original_bytes,+ observation.retained_bytes,+ ))+ .collect::<Vec<_>>(),+ vec![("transcript_image", "file_1".len(), 0)]+ );+ // Newer evidence evicts the full-budget ID; the oversized ID must not displace it.+ let content = [+ ImageReference::File {+ file_id: "x".repeat(MAX_TRANSCRIPT_IMAGE_BYTES),+ },+ ImageReference::Inline {+ image_url: "recent".into(),+ },+ ImageReference::File {+ file_id: "y".repeat(MAX_TRANSCRIPT_IMAGE_BYTES + 1),+ },+ ]+ .map(|image| ContentItem::InputImage {+ image,+ detail: Some(ImageDetail::High),+ });assert_eq!(TranscriptImages::collect(- &history,+ &[user_message(content.to_vec())],TranscriptImageInput {enabled: true,include_tool_outputs: true,- node_repl_images: &repl,- }+ node_repl_images: &[],+ },),TranscriptImages {- images: ["first", "second", "third", "fourth"].map(image).to_vec(),- omitted_bytes: 0,+ images: vec![image("recent")],+ omitted_bytes: 2 * MAX_TRANSCRIPT_IMAGE_BYTES + 1,});}