Lightweight coding agent that runs in your terminal
Rust
Keep image resize notices attached during remote compaction (#36956)
rka-oai4bd5b9fd0933
diff --git a/codex-rs/core/src/compact_remote.rs b/codex-rs/core/src/compact_remote.rsindex 26e783026f..5b515105d5 100644--- a/codex-rs/core/src/compact_remote.rs+++ b/codex-rs/core/src/compact_remote.rs@@ -10,6 +10,8 @@ use crate::compact::compaction_status_from_result;use crate::compact::insert_initial_context_before_last_real_user_or_summary;use crate::compact_model_fallback::record_model_fallback;use crate::compact_model_fallback::should_retry_with_current_model;+use crate::compact_remote_history::HistoryItemGroup;+use crate::compact_remote_history::history_item_groups;use crate::context::world_state::WorldState;use crate::context_manager::ContextManager;use crate::context_manager::estimate_item_token_count;@@ -301,7 +303,7 @@ async fn run_remote_compact_task_inner_impl(pub(crate) async fn process_compacted_history(sess: &Session,- mut compacted_history: Vec<ResponseItem>,+ compacted_history: Vec<ResponseItem>,initial_context_injection: &InitialContextInjection,) -> (Vec<ResponseItem>, Option<Arc<WorldState>>) {// Mid-turn compaction is the only path that must inject initial context above the last user@@ -310,7 +312,10 @@ pub(crate) async fn process_compacted_history(let (initial_context, world_state_baseline) =build_compaction_initial_context(sess, initial_context_injection).await;- compacted_history.retain(should_keep_compacted_history_item);+ let compacted_history = history_item_groups(compacted_history)+ .filter(|group| should_keep_compacted_history_item(&group.source))+ .flat_map(HistoryItemGroup::into_items)+ .collect();(insert_initial_context_before_last_real_user_or_summary(compacted_history, initial_context),world_state_baseline,@@ -375,34 +380,34 @@ pub(crate) fn trim_function_call_history_to_fit_context_window(let base_tokens =i128::try_from(approx_token_count(&base_instructions.text)).unwrap_or(i128::MAX);let original_items = history.raw_items();- let item_token_estimates = original_items- .iter()- .map(estimate_item_token_count)- .collect::<Vec<_>>();- let mut estimated_tokens = item_token_estimates- .iter()- .copied()- .map(i128::from)+ let mut estimated_tokens = history_item_groups(original_items)+ .map(|group| group.estimated_token_count()).fold(base_tokens, i128::saturating_add);let initial_estimated_tokens = i64::try_from(estimated_tokens).unwrap_or(i64::MAX);let mut rewritten_items = Vec::new();+ let mut consumed_items = 0;- for (item, item_tokens) in original_items.iter().zip(item_token_estimates).rev() {+ for group in history_item_groups(original_items)+ .collect::<Vec<_>>()+ .into_iter()+ .rev()+ {if i64::try_from(estimated_tokens).unwrap_or(i64::MAX) <= context_window {break;}- let Some(rewritten_item) = rewritten_output_for_context_window(item) else {+ let Some(rewritten_item) = rewritten_output_for_context_window(group.source) else {break;};estimated_tokens = estimated_tokens- .saturating_sub(i128::from(item_tokens))+ .saturating_sub(group.estimated_token_count()).saturating_add(i128::from(estimate_item_token_count(&rewritten_item)));+ consumed_items += 1 + usize::from(group.attached_notice.is_some());rewritten_items.push(rewritten_item);}let rewritten_outputs = rewritten_items.len();if rewritten_outputs > 0 {- let retained_len = original_items.len() - rewritten_outputs;+ let retained_len = original_items.len() - consumed_items;let mut items = original_items[..retained_len].to_vec();items.extend(rewritten_items.into_iter().rev());history.replace(items);diff --git a/codex-rs/core/src/compact_remote_history.rs b/codex-rs/core/src/compact_remote_history.rsnew file mode 100644index 0000000000..15706cd69d--- /dev/null+++ b/codex-rs/core/src/compact_remote_history.rs@@ -0,0 +1,59 @@+use std::borrow::Borrow;++use crate::context_manager::estimate_item_token_count;+use codex_protocol::models::ContentItem;+use codex_protocol::models::ResponseItem;++const ATTACHED_NOTICE_MARKERS: &[(&str, &str)] =+ &[("<image_resize_notice>", "</image_resize_notice>")];++#[derive(Debug, Clone, PartialEq)]+pub(crate) struct HistoryItemGroup<T> {+ pub(crate) source: T,+ pub(crate) attached_notice: Option<T>,+}++impl<T: Borrow<ResponseItem>> HistoryItemGroup<T> {+ pub(crate) fn into_items(self) -> impl Iterator<Item = T> {+ std::iter::once(self.source).chain(self.attached_notice)+ }++ pub(crate) fn estimated_token_count(&self) -> i128 {+ let source_tokens = i128::from(estimate_item_token_count(self.source.borrow()));+ let notice_tokens = self.attached_notice.as_ref().map_or(0, |notice| {+ i128::from(estimate_item_token_count(notice.borrow()))+ });+ source_tokens.saturating_add(notice_tokens)+ }+}++pub(crate) fn history_item_groups<I>(items: I) -> impl Iterator<Item = HistoryItemGroup<I::Item>>+where+ I: IntoIterator,+ I::Item: Borrow<ResponseItem>,+{+ let mut items = items.into_iter().peekable();+ std::iter::from_fn(move || {+ let source = items.next()?;+ let attached_notice = items.next_if(|notice| is_attached_notice(notice.borrow()));+ Some(HistoryItemGroup {+ source,+ attached_notice,+ })+ })+}++fn is_attached_notice(notice: &ResponseItem) -> bool {+ matches!(+ notice,+ ResponseItem::Message { role, content, .. }+ if role == "developer"+ && matches!(+ content.as_slice(),+ [ContentItem::InputText { text }]+ if ATTACHED_NOTICE_MARKERS.iter().any(|(opening, closing)| {+ text.starts_with(opening) && text.ends_with(closing)+ })+ )+ )+}diff --git a/codex-rs/core/src/compact_remote_v2.rs b/codex-rs/core/src/compact_remote_v2.rsindex 34f48513ac..995cd2b700 100644--- a/codex-rs/core/src/compact_remote_v2.rs+++ b/codex-rs/core/src/compact_remote_v2.rs@@ -13,6 +13,8 @@ use crate::compact_model_fallback::record_model_fallback;use crate::compact_model_fallback::should_retry_with_current_model;use crate::compact_remote::process_compacted_history;use crate::compact_remote::should_keep_compacted_history_item;+use crate::compact_remote_history::HistoryItemGroup;+use crate::compact_remote_history::history_item_groups;use crate::context_manager::estimate_item_token_count;use crate::hook_runtime::PostCompactHookOutcome;use crate::hook_runtime::PreCompactHookOutcome;@@ -443,10 +445,10 @@ fn build_v2_compacted_history(prompt_input: &[ResponseItem],compaction_output: ResponseItem,) -> (Vec<ResponseItem>, usize) {- let retained = prompt_input- .iter()- .filter(|item| is_retained_for_remote_compaction_v2(item))- .filter(|item| should_keep_compacted_history_item(item))+ let retained = history_item_groups(prompt_input)+ .filter(|group| is_retained_for_remote_compaction_v2(group.source))+ .filter(|group| should_keep_compacted_history_item(group.source))+ .flat_map(HistoryItemGroup::into_items).cloned().collect::<Vec<_>>();let mut retained =@@ -494,18 +496,37 @@ fn truncate_retained_messages_for_remote_compaction() -> Vec<ResponseItem> {let mut remaining = max_tokens;let mut truncated_reversed = Vec::with_capacity(items.len());- for item in items.into_iter().rev() {+ for group in history_item_groups(items)+ .collect::<Vec<_>>()+ .into_iter()+ .rev()+ {if remaining == 0 {continue;}- let token_count = message_text_token_count(&item).max(1);+ let notice_tokens = group+ .attached_notice+ .as_ref()+ .map_or(0, |notice| message_text_token_count(notice).max(1));+ let token_count = message_text_token_count(&group.source)+ .max(1)+ .saturating_add(notice_tokens);if token_count <= remaining {- truncated_reversed.push(item);+ if let Some(notice) = group.attached_notice {+ truncated_reversed.push(notice);+ }+ truncated_reversed.push(group.source);remaining = remaining.saturating_sub(token_count);- } else if let Some(truncated_item) =- truncate_message_text_to_token_budget(item, /*max_tokens*/ remaining)+ } else if remaining > notice_tokens+ && let Some(truncated_item) = truncate_message_text_to_token_budget(+ group.source,+ /*max_tokens*/ remaining - notice_tokens,+ ){+ if let Some(notice) = group.attached_notice {+ truncated_reversed.push(notice);+ }truncated_reversed.push(truncated_item);remaining = 0;}diff --git a/codex-rs/core/src/lib.rs b/codex-rs/core/src/lib.rsindex f1ba9c705e..4cecf46aed 100644--- a/codex-rs/core/src/lib.rs+++ b/codex-rs/core/src/lib.rs@@ -22,6 +22,7 @@ pub use turn_metadata::detached_memory_responses_metadata;mod codex_thread;mod compact_model_fallback;mod compact_remote;+mod compact_remote_history;mod compact_remote_v2;mod compact_token_budget;mod config_lock;diff --git a/codex-rs/core/tests/suite/compact_remote.rs b/codex-rs/core/tests/suite/compact_remote.rsindex 00efbb24c9..8a1c5ad16c 100644--- a/codex-rs/core/tests/suite/compact_remote.rs+++ b/codex-rs/core/tests/suite/compact_remote.rs@@ -28,6 +28,7 @@ use codex_protocol::models::ResponseItem;use codex_protocol::openai_models::InputModality;use codex_protocol::protocol::ConversationStartParams;use codex_protocol::protocol::EventMsg;+use codex_protocol::protocol::InitialHistory;use codex_protocol::protocol::InterAgentCommunication;use codex_protocol::protocol::ItemCompletedEvent;use codex_protocol::protocol::ItemStartedEvent;@@ -928,7 +929,53 @@ async fn remote_compact_v2_reuses_compaction_trigger_for_followups() -> Result<(}),).await?;- let codex = harness.test().codex.clone();+ let image_url = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR4nGNgYAAAAAMAASsJTYQAAAAASUVORK5CYII=";+ let user_notice = "<image_resize_notice>retained user image</image_resize_notice>";+ let tool_notice = "<image_resize_notice>discarded tool image</image_resize_notice>";+ let unlisted_notice = "<unlisted_notice>discarded developer notice</unlisted_notice>";+ let developer_message = |text| {+ json!({+ "type": "message",+ "role": "developer",+ "content": [{ "type": "input_text", "text": text }]+ })+ };+ let initial_history = [+ json!({+ "type": "message",+ "role": "user",+ "content": [+ { "type": "input_text", "text": "retained image source" },+ { "type": "input_image", "image_url": image_url }+ ]+ }),+ developer_message(user_notice),+ developer_message(unlisted_notice),+ json!({+ "type": "function_call",+ "name": "view_image",+ "arguments": "{}",+ "call_id": "image-call"+ }),+ json!({+ "type": "function_call_output",+ "call_id": "image-call",+ "output": [{ "type": "input_image", "image_url": image_url }]+ }),+ developer_message(tool_notice),+ ]+ .into_iter()+ .map(|item| serde_json::from_value(item).map(RolloutItem::ResponseItem))+ .collect::<serde_json::Result<Vec<_>>>()?;+ let codex = harness+ .test()+ .thread_manager+ .start_thread(StartThreadOptions {+ initial_history: InitialHistory::Forked(initial_history),+ ..StartThreadOptions::new(harness.test().config.clone())+ })+ .await?+ .thread;let responses_mock = responses::mount_sse_sequence(harness.server(),@@ -1109,6 +1156,23 @@ async fn remote_compact_v2_reuses_compaction_trigger_for_followups() -> Result<(follow_up_body.contains("hello remote compact"),"expected v2 follow-up request to preserve retained original user messages");+ assert!(+ follow_up_request.input().windows(2).any(|items| {+ items[0]["role"] == "user"+ && items[0]["content"][0]["text"] == "retained image source"+ && items[1]["role"] == "developer"+ && items[1]["content"][0]["text"] == user_notice+ }),+ "expected v2 compaction to retain the user image and its adjacent resize notice"+ );+ assert!(+ !follow_up_body.contains(unlisted_notice),+ "expected v2 compaction to drop unlisted developer notices"+ );+ assert!(+ !follow_up_body.contains(tool_notice),+ "expected v2 compaction to drop the resize notice with its tool output"+ );Ok(())}