Lightweight coding agent that runs in your terminal
Rust
Include images in Guardian v2 transcripts (#38987)
jif37cf6c84c086
diff --git a/codex-rs/core/config.schema.json b/codex-rs/core/config.schema.jsonindex 371b031ad8..2addb50e8e 100644--- a/codex-rs/core/config.schema.json+++ b/codex-rs/core/config.schema.json@@ -1276,6 +1276,10 @@"additionalProperties": false,"description": "Bounds and optional sources for the Guardian v2 conversation transcript.","properties": {+ "include_images": {+ "description": "Include recent screenshots from messages and configured tool outputs.",+ "type": "boolean"+ },"max_message_entry_tokens": {"format": "uint","maximum": 100000.0,diff --git a/codex-rs/ext/guardian-v2/src/config.rs b/codex-rs/ext/guardian-v2/src/config.rsindex 506d2e48bb..0dc35a8758 100644--- a/codex-rs/ext/guardian-v2/src/config.rs+++ b/codex-rs/ext/guardian-v2/src/config.rs@@ -144,6 +144,9 @@ impl GuardianV2Config {.unwrap_or_else(|| {vec![TranscriptSource::ToolCalls, TranscriptSource::ToolOutputs]}),+ include_images: transcript_config+ .and_then(|transcript| transcript.include_images)+ .unwrap_or(false),max_message_entry_tokens,max_tool_entry_tokens,max_message_transcript_tokens,diff --git a/codex-rs/ext/guardian-v2/src/extension.rs b/codex-rs/ext/guardian-v2/src/extension.rsindex a38a47528a..2834fbac4d 100644--- a/codex-rs/ext/guardian-v2/src/extension.rs+++ b/codex-rs/ext/guardian-v2/src/extension.rs@@ -350,6 +350,9 @@ impl ToolLifecycleContributor for GuardianV2Extension {let transcript = guardian_config.transcript.build(conversation_history.items());+ let images = guardian_config+ .transcript+ .images(conversation_history.items());drop(conversation_history);let planned_action = match action.render(guardian_config.max_action_tokens) {Ok(planned_action) => planned_action,@@ -418,6 +421,7 @@ impl ToolLifecycleContributor for GuardianV2Extension {.sample(LunaSamplingRequest {instructions,input: classification_input,+ images,parent_compaction,parent_compaction_hash,output_schema: json!({diff --git a/codex-rs/ext/guardian-v2/src/extension_tests.rs b/codex-rs/ext/guardian-v2/src/extension_tests.rsindex b5963933d2..d04ca355e7 100644--- a/codex-rs/ext/guardian-v2/src/extension_tests.rs+++ b/codex-rs/ext/guardian-v2/src/extension_tests.rs@@ -21,7 +21,9 @@ use codex_login::CodexAuth;use codex_model_provider_info::ModelProviderInfo;use codex_protocol::ResponseItemId;use codex_protocol::models::ContentItem;+use codex_protocol::models::FunctionCallOutputContentItem;use codex_protocol::models::FunctionCallOutputPayload;+use codex_protocol::models::ImageDetail;use codex_protocol::models::InternalChatMessageMetadataPassthrough;use codex_protocol::models::ReasoningItemReasoningSummary;use codex_protocol::protocol::ReviewDecision;@@ -459,6 +461,76 @@ max_recent_non_user_entries = 8Ok(())}+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]+async fn contributor_includes_configured_transcript_images() -> Result<()> {+ skip_if_no_network!(Ok(()));++ let history = vec![+ ResponseItem::Message {+ id: None,+ role: "user".to_owned(),+ content: vec![+ ContentItem::InputText {+ text: "Review what is shown on screen.".to_owned(),+ },+ ContentItem::InputImage {+ image_url: "data:image/png;base64,user-screenshot".to_owned(),+ detail: Some(ImageDetail::High),+ },+ ],+ phase: None,+ internal_chat_message_metadata_passthrough: None,+ },+ ResponseItem::FunctionCallOutput {+ id: None,+ call_id: "previous-call".to_owned(),+ output: FunctionCallOutputPayload::from_content_items(vec![+ FunctionCallOutputContentItem::InputText {+ text: "Screenshot captured.".to_owned(),+ },+ FunctionCallOutputContentItem::InputImage {+ image_url: "data:image/png;base64,tool-screenshot".to_owned(),+ detail: Some(ImageDetail::Low),+ },+ ]),+ internal_chat_message_metadata_passthrough: None,+ },+ ];+ let configuration = r#"+[features.guardianv2]+enabled = true++[features.guardianv2.transcript]+include_images = true+"#;+ let (request, _test, _registry) = sample_configured_conversation_history(+ history,+ r#"{"path":"README.md"}"#,+ Some(TEST_GUARDIAN_POLICY),+ configuration,+ )+ .await?;+ let content = request["input"][2]["content"]+ .as_array()+ .expect("Luna user content should be an array");++ assert_eq!(+ content[content.len() - 2..],+ [+ json!({+ "type": "input_image",+ "image_url": "data:image/png;base64,user-screenshot",+ }),+ json!({+ "type": "input_image",+ "image_url": "data:image/png;base64,tool-screenshot",+ }),+ ]+ );++ Ok(())+}+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]async fn contributor_samples_tool_calls_with_the_existing_luna_pool() -> Result<()> {skip_if_no_network!(Ok(()));diff --git a/codex-rs/ext/guardian-v2/src/sampler.rs b/codex-rs/ext/guardian-v2/src/sampler.rsindex a02acccb70..358d0a9724 100644--- a/codex-rs/ext/guardian-v2/src/sampler.rs+++ b/codex-rs/ext/guardian-v2/src/sampler.rs@@ -74,6 +74,8 @@ pub struct LunaSamplingRequest {pub instructions: String,/// Ordered untrusted input entries that the model should classify.pub input: Vec<String>,+ /// Optional bounded screenshots accompanying the transcript.+ pub images: Vec<ContentItem>,/// Opaque parent compaction to reuse only for compatible model configurations.pub parent_compaction: Option<ResponseItem>,/// Current parent model's encrypted-compaction compatibility hash.@@ -317,6 +319,12 @@ impl LunaSampler {.input.into_iter().map(|text| ContentItem::InputText { text })+ .chain(request.images.into_iter().map(|mut image| {+ if let ContentItem::InputImage { detail, .. } = &mut image {+ *detail = None;+ }+ image+ })).collect(),phase: None,internal_chat_message_metadata_passthrough: None,diff --git a/codex-rs/ext/guardian-v2/src/sampler_tests.rs b/codex-rs/ext/guardian-v2/src/sampler_tests.rsindex 5e27d62c9f..c4b55b6fef 100644--- a/codex-rs/ext/guardian-v2/src/sampler_tests.rs+++ b/codex-rs/ext/guardian-v2/src/sampler_tests.rs@@ -75,6 +75,7 @@ fn sample_request(turn_id: &str) -> LunaSamplingRequest {LunaSamplingRequest {instructions: "Return a risk score.".to_owned(),input: vec!["The user requested a README summary.".to_owned()],+ images: Vec::new(),parent_compaction: None,parent_compaction_hash: None,output_schema: json!({@@ -169,6 +170,7 @@ async fn preconnected_sampler_reuses_authenticated_websocket_for_structured_requ"The user requested a README summary.".to_owned(),"The assistant inspected README.md.".to_owned(),],+ images: Vec::new(),parent_compaction: None,parent_compaction_hash: None,output_schema: schema.clone(),@@ -192,6 +194,7 @@ async fn preconnected_sampler_reuses_authenticated_websocket_for_structured_requ.sample(LunaSamplingRequest {instructions: "Return a risk score.".to_owned(),input: vec!["The user requested a source review.".to_owned()],+ images: Vec::new(),parent_compaction: None,parent_compaction_hash: None,output_schema: schema,@@ -342,6 +345,7 @@ async fn sampler_returns_complete_json_before_terminal_response_events() -> Resusampler.sample(LunaSamplingRequest {instructions: "Return a risk score.".to_owned(),input: vec!["The user requested a README summary.".to_owned()],+ images: Vec::new(),parent_compaction: None,parent_compaction_hash: None,output_schema: json!({diff --git a/codex-rs/ext/guardian-v2/src/transcript.rs b/codex-rs/ext/guardian-v2/src/transcript.rsindex 4262e57ecd..10f9ac2f8b 100644--- a/codex-rs/ext/guardian-v2/src/transcript.rs+++ b/codex-rs/ext/guardian-v2/src/transcript.rs@@ -1,8 +1,11 @@use std::collections::HashMap;+use std::collections::VecDeque;use codex_extension_api::ResponseItem;pub(crate) use codex_features::GuardianV2TranscriptSource as TranscriptSource;use codex_protocol::models::ContentItem;+use codex_protocol::models::FunctionCallOutputContentItem;+use codex_protocol::models::ImageDetail;use codex_protocol::models::ReasoningItemContent;use codex_protocol::models::ReasoningItemReasoningSummary;use codex_protocol::models::plaintext_agent_message_content;@@ -13,6 +16,8 @@ pub(crate) const MAX_TOOL_ENTRY_TOKENS: usize = 1_000;pub(crate) const MAX_MESSAGE_TRANSCRIPT_TOKENS: usize = 10_000;pub(crate) const MAX_TOOL_TRANSCRIPT_TOKENS: usize = 10_000;pub(crate) const MAX_RECENT_NON_USER_ENTRIES: usize = 40;+const MAX_TRANSCRIPT_IMAGES: usize = 4;+const MAX_TRANSCRIPT_IMAGE_BYTES: usize = 8 * 1024 * 1024;const MANUAL_APPROVAL_DEVELOPER_PREFIX: &str ="The user has manually approved a specific action that was previously `Rejected`.";@@ -32,6 +37,7 @@ struct TranscriptEntry {#[derive(Clone, Debug, Eq, PartialEq)]pub(crate) struct TranscriptConfig {pub(crate) sources: Vec<TranscriptSource>,+ pub(crate) include_images: bool,pub(crate) max_message_entry_tokens: usize,pub(crate) max_tool_entry_tokens: usize,pub(crate) max_message_transcript_tokens: usize,@@ -43,6 +49,7 @@ impl Default for TranscriptConfig {fn default() -> Self {Self {sources: vec![TranscriptSource::ToolCalls, TranscriptSource::ToolOutputs],+ include_images: false,max_message_entry_tokens: MAX_MESSAGE_ENTRY_TOKENS,max_tool_entry_tokens: MAX_TOOL_ENTRY_TOKENS,max_message_transcript_tokens: MAX_MESSAGE_TRANSCRIPT_TOKENS,@@ -53,6 +60,67 @@ impl Default for TranscriptConfig {}impl TranscriptConfig {+ pub(crate) fn images<'a>(+ &self,+ items: impl IntoIterator<Item = &'a ResponseItem>,+ ) -> Vec<ContentItem> {+ if !self.include_images {+ return Vec::new();+ }++ let mut images = VecDeque::new();+ let mut image_bytes = 0usize;+ let mut include_image = |image_url: &str, detail: Option<ImageDetail>| {+ if image_url.len() > MAX_TRANSCRIPT_IMAGE_BYTES {+ return;+ }+ while images.len() >= MAX_TRANSCRIPT_IMAGES+ || image_bytes + image_url.len() > MAX_TRANSCRIPT_IMAGE_BYTES+ {+ let Some(ContentItem::InputImage { image_url, .. }) = images.pop_front() else {+ break;+ };+ image_bytes -= image_url.len();+ }+ image_bytes += image_url.len();+ images.push_back(ContentItem::InputImage {+ image_url: image_url.to_owned(),+ detail,+ });+ };++ for item in items {+ match item {+ ResponseItem::Message { role, content, .. }+ if matches!(role.as_str(), "user" | "assistant") =>+ {+ for item in content {+ if let ContentItem::InputImage { image_url, detail } = item {+ include_image(image_url, *detail);+ }+ }+ }+ ResponseItem::FunctionCallOutput { output, .. }+ | ResponseItem::CustomToolCallOutput { output, .. }+ if self.sources.contains(&TranscriptSource::ToolOutputs) =>+ {+ if let Some(content) = output.content_items() {+ for item in content {+ if let FunctionCallOutputContentItem::InputImage { image_url, detail } =+ item+ {+ include_image(image_url, *detail);+ }+ }+ }+ }+ _ => {}+ }+ }++ images.into_iter().collect()+ }+pub(crate) fn build<'a>(&self,items: impl IntoIterator<Item = &'a ResponseItem>,diff --git a/codex-rs/features/src/feature_configs.rs b/codex-rs/features/src/feature_configs.rsindex 24a92ebb2c..2c2d385529 100644--- a/codex-rs/features/src/feature_configs.rs+++ b/codex-rs/features/src/feature_configs.rs@@ -88,6 +88,9 @@ pub enum GuardianV2TranscriptSource {pub struct GuardianV2TranscriptConfigToml {#[serde(skip_serializing_if = "Option::is_none")]pub sources: Option<Vec<GuardianV2TranscriptSource>>,+ /// Include recent screenshots from messages and configured tool outputs.+ #[serde(skip_serializing_if = "Option::is_none")]+ pub include_images: Option<bool>,#[serde(skip_serializing_if = "Option::is_none")]#[schemars(range(min = 100, max = 100000))]pub max_message_entry_tokens: Option<usize>,diff --git a/codex-rs/features/src/tests.rs b/codex-rs/features/src/tests.rsindex 054f5d7aff..4c5003899e 100644--- a/codex-rs/features/src/tests.rs+++ b/codex-rs/features/src/tests.rs@@ -143,6 +143,7 @@ max_parent_compaction_tokens = 384[guardianv2.transcript]sources = ["tool_outputs", "reasoning"]+include_images = truemax_message_entry_tokens = 128max_tool_entry_tokens = 128max_message_transcript_tokens = 512@@ -167,6 +168,7 @@ max_recent_non_user_entries = 12crate::GuardianV2TranscriptSource::ToolOutputs,crate::GuardianV2TranscriptSource::Reasoning,]),+ include_images: Some(true),max_message_entry_tokens: Some(128),max_tool_entry_tokens: Some(128),max_message_transcript_tokens: Some(512),