Skip to content

armgit / codex

Public Imported from GitHub
0

Lightweight coding agent that runs in your terminal

Rust

Include images in Guardian v2 transcripts (#38987)

jif37cf6c84c086
diff --git a/codex-rs/core/config.schema.json b/codex-rs/core/config.schema.json
index 371b031ad8..2addb50e8e 100644
--- a/codex-rs/core/config.schema.json
+++ b/codex-rs/core/config.schema.json
@@ -1276,6 +1276,10 @@
"additionalProperties": false,
"description": "Bounds and optional sources for the Guardian v2 conversation transcript.",
"properties": {
+ "include_images": {
+ "description": "Include recent screenshots from messages and configured tool outputs.",
+ "type": "boolean"
+ },
"max_message_entry_tokens": {
"format": "uint",
"maximum": 100000.0,
diff --git a/codex-rs/ext/guardian-v2/src/config.rs b/codex-rs/ext/guardian-v2/src/config.rs
index 506d2e48bb..0dc35a8758 100644
--- a/codex-rs/ext/guardian-v2/src/config.rs
+++ b/codex-rs/ext/guardian-v2/src/config.rs
@@ -144,6 +144,9 @@ impl GuardianV2Config {
.unwrap_or_else(|| {
vec![TranscriptSource::ToolCalls, TranscriptSource::ToolOutputs]
}),
+ include_images: transcript_config
+ .and_then(|transcript| transcript.include_images)
+ .unwrap_or(false),
max_message_entry_tokens,
max_tool_entry_tokens,
max_message_transcript_tokens,
diff --git a/codex-rs/ext/guardian-v2/src/extension.rs b/codex-rs/ext/guardian-v2/src/extension.rs
index a38a47528a..2834fbac4d 100644
--- a/codex-rs/ext/guardian-v2/src/extension.rs
+++ b/codex-rs/ext/guardian-v2/src/extension.rs
@@ -350,6 +350,9 @@ impl ToolLifecycleContributor for GuardianV2Extension {
let transcript = guardian_config
.transcript
.build(conversation_history.items());
+ let images = guardian_config
+ .transcript
+ .images(conversation_history.items());
drop(conversation_history);
let planned_action = match action.render(guardian_config.max_action_tokens) {
Ok(planned_action) => planned_action,
@@ -418,6 +421,7 @@ impl ToolLifecycleContributor for GuardianV2Extension {
.sample(LunaSamplingRequest {
instructions,
input: classification_input,
+ images,
parent_compaction,
parent_compaction_hash,
output_schema: json!({
diff --git a/codex-rs/ext/guardian-v2/src/extension_tests.rs b/codex-rs/ext/guardian-v2/src/extension_tests.rs
index b5963933d2..d04ca355e7 100644
--- a/codex-rs/ext/guardian-v2/src/extension_tests.rs
+++ b/codex-rs/ext/guardian-v2/src/extension_tests.rs
@@ -21,7 +21,9 @@ use codex_login::CodexAuth;
use codex_model_provider_info::ModelProviderInfo;
use codex_protocol::ResponseItemId;
use codex_protocol::models::ContentItem;
+use codex_protocol::models::FunctionCallOutputContentItem;
use codex_protocol::models::FunctionCallOutputPayload;
+use codex_protocol::models::ImageDetail;
use codex_protocol::models::InternalChatMessageMetadataPassthrough;
use codex_protocol::models::ReasoningItemReasoningSummary;
use codex_protocol::protocol::ReviewDecision;
@@ -459,6 +461,76 @@ max_recent_non_user_entries = 8
Ok(())
}
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn contributor_includes_configured_transcript_images() -> Result<()> {
+ skip_if_no_network!(Ok(()));
+
+ let history = vec![
+ ResponseItem::Message {
+ id: None,
+ role: "user".to_owned(),
+ content: vec![
+ ContentItem::InputText {
+ text: "Review what is shown on screen.".to_owned(),
+ },
+ ContentItem::InputImage {
+ image_url: "data:image/png;base64,user-screenshot".to_owned(),
+ detail: Some(ImageDetail::High),
+ },
+ ],
+ phase: None,
+ internal_chat_message_metadata_passthrough: None,
+ },
+ ResponseItem::FunctionCallOutput {
+ id: None,
+ call_id: "previous-call".to_owned(),
+ output: FunctionCallOutputPayload::from_content_items(vec![
+ FunctionCallOutputContentItem::InputText {
+ text: "Screenshot captured.".to_owned(),
+ },
+ FunctionCallOutputContentItem::InputImage {
+ image_url: "data:image/png;base64,tool-screenshot".to_owned(),
+ detail: Some(ImageDetail::Low),
+ },
+ ]),
+ internal_chat_message_metadata_passthrough: None,
+ },
+ ];
+ let configuration = r#"
+[features.guardianv2]
+enabled = true
+
+[features.guardianv2.transcript]
+include_images = true
+"#;
+ let (request, _test, _registry) = sample_configured_conversation_history(
+ history,
+ r#"{"path":"README.md"}"#,
+ Some(TEST_GUARDIAN_POLICY),
+ configuration,
+ )
+ .await?;
+ let content = request["input"][2]["content"]
+ .as_array()
+ .expect("Luna user content should be an array");
+
+ assert_eq!(
+ content[content.len() - 2..],
+ [
+ json!({
+ "type": "input_image",
+ "image_url": "data:image/png;base64,user-screenshot",
+ }),
+ json!({
+ "type": "input_image",
+ "image_url": "data:image/png;base64,tool-screenshot",
+ }),
+ ]
+ );
+
+ Ok(())
+}
+
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn contributor_samples_tool_calls_with_the_existing_luna_pool() -> Result<()> {
skip_if_no_network!(Ok(()));
diff --git a/codex-rs/ext/guardian-v2/src/sampler.rs b/codex-rs/ext/guardian-v2/src/sampler.rs
index a02acccb70..358d0a9724 100644
--- a/codex-rs/ext/guardian-v2/src/sampler.rs
+++ b/codex-rs/ext/guardian-v2/src/sampler.rs
@@ -74,6 +74,8 @@ pub struct LunaSamplingRequest {
pub instructions: String,
/// Ordered untrusted input entries that the model should classify.
pub input: Vec<String>,
+ /// Optional bounded screenshots accompanying the transcript.
+ pub images: Vec<ContentItem>,
/// Opaque parent compaction to reuse only for compatible model configurations.
pub parent_compaction: Option<ResponseItem>,
/// Current parent model's encrypted-compaction compatibility hash.
@@ -317,6 +319,12 @@ impl LunaSampler {
.input
.into_iter()
.map(|text| ContentItem::InputText { text })
+ .chain(request.images.into_iter().map(|mut image| {
+ if let ContentItem::InputImage { detail, .. } = &mut image {
+ *detail = None;
+ }
+ image
+ }))
.collect(),
phase: None,
internal_chat_message_metadata_passthrough: None,
diff --git a/codex-rs/ext/guardian-v2/src/sampler_tests.rs b/codex-rs/ext/guardian-v2/src/sampler_tests.rs
index 5e27d62c9f..c4b55b6fef 100644
--- a/codex-rs/ext/guardian-v2/src/sampler_tests.rs
+++ b/codex-rs/ext/guardian-v2/src/sampler_tests.rs
@@ -75,6 +75,7 @@ fn sample_request(turn_id: &str) -> LunaSamplingRequest {
LunaSamplingRequest {
instructions: "Return a risk score.".to_owned(),
input: vec!["The user requested a README summary.".to_owned()],
+ images: Vec::new(),
parent_compaction: None,
parent_compaction_hash: None,
output_schema: json!({
@@ -169,6 +170,7 @@ async fn preconnected_sampler_reuses_authenticated_websocket_for_structured_requ
"The user requested a README summary.".to_owned(),
"The assistant inspected README.md.".to_owned(),
],
+ images: Vec::new(),
parent_compaction: None,
parent_compaction_hash: None,
output_schema: schema.clone(),
@@ -192,6 +194,7 @@ async fn preconnected_sampler_reuses_authenticated_websocket_for_structured_requ
.sample(LunaSamplingRequest {
instructions: "Return a risk score.".to_owned(),
input: vec!["The user requested a source review.".to_owned()],
+ images: Vec::new(),
parent_compaction: None,
parent_compaction_hash: None,
output_schema: schema,
@@ -342,6 +345,7 @@ async fn sampler_returns_complete_json_before_terminal_response_events() -> Resu
sampler.sample(LunaSamplingRequest {
instructions: "Return a risk score.".to_owned(),
input: vec!["The user requested a README summary.".to_owned()],
+ images: Vec::new(),
parent_compaction: None,
parent_compaction_hash: None,
output_schema: json!({
diff --git a/codex-rs/ext/guardian-v2/src/transcript.rs b/codex-rs/ext/guardian-v2/src/transcript.rs
index 4262e57ecd..10f9ac2f8b 100644
--- a/codex-rs/ext/guardian-v2/src/transcript.rs
+++ b/codex-rs/ext/guardian-v2/src/transcript.rs
@@ -1,8 +1,11 @@
use std::collections::HashMap;
+use std::collections::VecDeque;
use codex_extension_api::ResponseItem;
pub(crate) use codex_features::GuardianV2TranscriptSource as TranscriptSource;
use codex_protocol::models::ContentItem;
+use codex_protocol::models::FunctionCallOutputContentItem;
+use codex_protocol::models::ImageDetail;
use codex_protocol::models::ReasoningItemContent;
use codex_protocol::models::ReasoningItemReasoningSummary;
use codex_protocol::models::plaintext_agent_message_content;
@@ -13,6 +16,8 @@ pub(crate) const MAX_TOOL_ENTRY_TOKENS: usize = 1_000;
pub(crate) const MAX_MESSAGE_TRANSCRIPT_TOKENS: usize = 10_000;
pub(crate) const MAX_TOOL_TRANSCRIPT_TOKENS: usize = 10_000;
pub(crate) const MAX_RECENT_NON_USER_ENTRIES: usize = 40;
+const MAX_TRANSCRIPT_IMAGES: usize = 4;
+const MAX_TRANSCRIPT_IMAGE_BYTES: usize = 8 * 1024 * 1024;
const MANUAL_APPROVAL_DEVELOPER_PREFIX: &str =
"The user has manually approved a specific action that was previously `Rejected`.";
@@ -32,6 +37,7 @@ struct TranscriptEntry {
#[derive(Clone, Debug, Eq, PartialEq)]
pub(crate) struct TranscriptConfig {
pub(crate) sources: Vec<TranscriptSource>,
+ pub(crate) include_images: bool,
pub(crate) max_message_entry_tokens: usize,
pub(crate) max_tool_entry_tokens: usize,
pub(crate) max_message_transcript_tokens: usize,
@@ -43,6 +49,7 @@ impl Default for TranscriptConfig {
fn default() -> Self {
Self {
sources: vec![TranscriptSource::ToolCalls, TranscriptSource::ToolOutputs],
+ include_images: false,
max_message_entry_tokens: MAX_MESSAGE_ENTRY_TOKENS,
max_tool_entry_tokens: MAX_TOOL_ENTRY_TOKENS,
max_message_transcript_tokens: MAX_MESSAGE_TRANSCRIPT_TOKENS,
@@ -53,6 +60,67 @@ impl Default for TranscriptConfig {
}
impl TranscriptConfig {
+ pub(crate) fn images<'a>(
+ &self,
+ items: impl IntoIterator<Item = &'a ResponseItem>,
+ ) -> Vec<ContentItem> {
+ if !self.include_images {
+ return Vec::new();
+ }
+
+ let mut images = VecDeque::new();
+ let mut image_bytes = 0usize;
+ let mut include_image = |image_url: &str, detail: Option<ImageDetail>| {
+ if image_url.len() > MAX_TRANSCRIPT_IMAGE_BYTES {
+ return;
+ }
+ while images.len() >= MAX_TRANSCRIPT_IMAGES
+ || image_bytes + image_url.len() > MAX_TRANSCRIPT_IMAGE_BYTES
+ {
+ let Some(ContentItem::InputImage { image_url, .. }) = images.pop_front() else {
+ break;
+ };
+ image_bytes -= image_url.len();
+ }
+ image_bytes += image_url.len();
+ images.push_back(ContentItem::InputImage {
+ image_url: image_url.to_owned(),
+ detail,
+ });
+ };
+
+ for item in items {
+ match item {
+ ResponseItem::Message { role, content, .. }
+ if matches!(role.as_str(), "user" | "assistant") =>
+ {
+ for item in content {
+ if let ContentItem::InputImage { image_url, detail } = item {
+ include_image(image_url, *detail);
+ }
+ }
+ }
+ ResponseItem::FunctionCallOutput { output, .. }
+ | ResponseItem::CustomToolCallOutput { output, .. }
+ if self.sources.contains(&TranscriptSource::ToolOutputs) =>
+ {
+ if let Some(content) = output.content_items() {
+ for item in content {
+ if let FunctionCallOutputContentItem::InputImage { image_url, detail } =
+ item
+ {
+ include_image(image_url, *detail);
+ }
+ }
+ }
+ }
+ _ => {}
+ }
+ }
+
+ images.into_iter().collect()
+ }
+
pub(crate) fn build<'a>(
&self,
items: impl IntoIterator<Item = &'a ResponseItem>,
diff --git a/codex-rs/features/src/feature_configs.rs b/codex-rs/features/src/feature_configs.rs
index 24a92ebb2c..2c2d385529 100644
--- a/codex-rs/features/src/feature_configs.rs
+++ b/codex-rs/features/src/feature_configs.rs
@@ -88,6 +88,9 @@ pub enum GuardianV2TranscriptSource {
pub struct GuardianV2TranscriptConfigToml {
#[serde(skip_serializing_if = "Option::is_none")]
pub sources: Option<Vec<GuardianV2TranscriptSource>>,
+ /// Include recent screenshots from messages and configured tool outputs.
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub include_images: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
#[schemars(range(min = 100, max = 100000))]
pub max_message_entry_tokens: Option<usize>,
diff --git a/codex-rs/features/src/tests.rs b/codex-rs/features/src/tests.rs
index 054f5d7aff..4c5003899e 100644
--- a/codex-rs/features/src/tests.rs
+++ b/codex-rs/features/src/tests.rs
@@ -143,6 +143,7 @@ max_parent_compaction_tokens = 384
[guardianv2.transcript]
sources = ["tool_outputs", "reasoning"]
+include_images = true
max_message_entry_tokens = 128
max_tool_entry_tokens = 128
max_message_transcript_tokens = 512
@@ -167,6 +168,7 @@ max_recent_non_user_entries = 12
crate::GuardianV2TranscriptSource::ToolOutputs,
crate::GuardianV2TranscriptSource::Reasoning,
]),
+ include_images: Some(true),
max_message_entry_tokens: Some(128),
max_tool_entry_tokens: Some(128),
max_message_transcript_tokens: Some(512),