Skip to content

armgit / codex

Public Imported from GitHub
0

Lightweight coding agent that runs in your terminal

Rust

Include node_repl screenshots in Guardian v2 reviews (#39227)

jif9b9b614b02ba
diff --git a/codex-rs/core/src/context/mod.rs b/codex-rs/core/src/context/mod.rs
index 4f69daa3ba..e41efb06d4 100644
--- a/codex-rs/core/src/context/mod.rs
+++ b/codex-rs/core/src/context/mod.rs
@@ -67,7 +67,7 @@ pub(crate) use model_switch_instructions::ModelSwitchInstructions;
pub(crate) use multi_agent_role_instructions::MultiAgentRoleInstructions;
pub(crate) use multi_agent_usage_hint::MultiAgentUsageHint;
pub(crate) use network_rule_saved::NetworkRuleSaved;
-pub(crate) use node_repl_review_evidence::NodeReplReviewEvidence;
+pub use node_repl_review_evidence::NodeReplReviewEvidence;
pub(crate) use node_repl_review_evidence::NodeReplReviewEvidenceMode;
pub(crate) use node_repl_review_evidence::node_repl_review_evidence_mode;
pub use permissions_instructions::ApprovalPromptContext;
diff --git a/codex-rs/core/src/context/node_repl_review_evidence.rs b/codex-rs/core/src/context/node_repl_review_evidence.rs
index 29f884ea78..e68b17dc5b 100644
--- a/codex-rs/core/src/context/node_repl_review_evidence.rs
+++ b/codex-rs/core/src/context/node_repl_review_evidence.rs
@@ -5,6 +5,7 @@ use std::sync::Mutex;
use std::sync::PoisonError;
use codex_features::Feature;
+use codex_protocol::models::ContentItem;
use codex_protocol::user_input::UserInput;
use codex_protocol::user_input::UserInput::Image;
use codex_protocol::user_input::UserInput::Text;
@@ -84,14 +85,51 @@ struct NodeReplReviewEvidenceState {
responses: VecDeque<Arc<NodeReplReviewResponse>>,
next_sequence: u64,
retained_bytes: usize,
+ image_capture_enabled: bool,
}
+/// Bounded, thread-scoped evidence collected from nested `node_repl` calls.
#[derive(Debug, Default)]
-pub(crate) struct NodeReplReviewEvidence(Mutex<NodeReplReviewEvidenceState>);
+pub struct NodeReplReviewEvidence(Mutex<NodeReplReviewEvidenceState>);
impl NodeReplReviewEvidence {
pub(crate) const MAX_RETAINED_BYTES: usize = 8 * 1024 * 1024;
+ /// Enables screenshot capture even when synchronous Guardian transcripts are disabled.
+ pub fn enable_image_capture(&self) {
+ self.0
+ .lock()
+ .unwrap_or_else(PoisonError::into_inner)
+ .image_capture_enabled = true;
+ }
+
+ pub(crate) fn image_capture_enabled(&self) -> bool {
+ self.0
+ .lock()
+ .unwrap_or_else(PoisonError::into_inner)
+ .image_capture_enabled
+ }
+
+ /// Returns the screenshots currently retained for this thread, oldest first.
+ pub fn images(&self) -> Vec<ContentItem> {
+ let state = self.0.lock().unwrap_or_else(PoisonError::into_inner);
+ let mut seen_images = HashSet::new();
+ state
+ .responses
+ .iter()
+ .flat_map(|response| &response.items)
+ .filter_map(|item| match item {
+ Image { image_url, detail } if seen_images.insert(image_url) => {
+ Some(ContentItem::InputImage {
+ image_url: image_url.clone(),
+ detail: *detail,
+ })
+ }
+ _ => None,
+ })
+ .collect()
+ }
+
pub(crate) fn record(
&self,
tool_name: &str,
diff --git a/codex-rs/core/src/tools/handlers/mcp.rs b/codex-rs/core/src/tools/handlers/mcp.rs
index 02a94f4155..06de617093 100644
--- a/codex-rs/core/src/tools/handlers/mcp.rs
+++ b/codex-rs/core/src/tools/handlers/mcp.rs
@@ -250,9 +250,15 @@ impl CoreToolRuntime for McpHandler {
return;
};
let evidence_mode = node_repl_review_evidence_mode(&invocation.turn);
+ let image_capture_enabled = invocation
+ .session
+ .services
+ .thread_extension_data
+ .get::<NodeReplReviewEvidence>()
+ .is_some_and(|evidence| evidence.image_capture_enabled());
if self.tool_info.server_name != "node_repl"
|| !result.success_for_logging()
- || evidence_mode == NodeReplReviewEvidenceMode::Disabled
+ || evidence_mode == NodeReplReviewEvidenceMode::Disabled && !image_capture_enabled
{
return;
}
@@ -283,7 +289,10 @@ impl CoreToolRuntime for McpHandler {
text: text.to_string(),
text_elements: Vec::new(),
}),
- Some("image") if evidence_mode == NodeReplReviewEvidenceMode::Multimodal => {
+ Some("image")
+ if evidence_mode == NodeReplReviewEvidenceMode::Multimodal
+ || image_capture_enabled =>
+ {
let payload = item.get("data").and_then(Value::as_str)?;
let mime_type = item.get("mimeType").and_then(Value::as_str)?;
if payload.is_empty()
diff --git a/codex-rs/core/tests/suite/code_mode.rs b/codex-rs/core/tests/suite/code_mode.rs
index f9429a1464..f83353899c 100644
--- a/codex-rs/core/tests/suite/code_mode.rs
+++ b/codex-rs/core/tests/suite/code_mode.rs
@@ -10,6 +10,7 @@ use codex_core::TurnInputRequest;
use codex_core::config::Config;
use codex_core::config::Constrained;
use codex_core::config::CurrentTimeReminderConfig;
+use codex_core::context::NodeReplReviewEvidence;
use codex_extension_api::ExtensionData;
use codex_extension_api::ExtensionRegistryBuilder;
use codex_extension_api::ToolCallOutcome;
@@ -33,6 +34,8 @@ use codex_protocol::dynamic_tools::DynamicToolNamespaceSpec;
use codex_protocol::dynamic_tools::DynamicToolNamespaceTool;
use codex_protocol::dynamic_tools::DynamicToolResponse;
use codex_protocol::dynamic_tools::DynamicToolSpec;
+use codex_protocol::models::ContentItem;
+use codex_protocol::models::ImageDetail;
use codex_protocol::models::PermissionProfile;
use codex_protocol::models::ResponseItem;
use codex_protocol::openai_models::ToolMode;
@@ -4344,6 +4347,77 @@ contentLength=0"
Ok(())
}
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn code_mode_node_repl_screenshots_can_be_captured_without_guardian_transcript_flags()
+-> Result<()> {
+ skip_if_wine_exec!(
+ Ok(()),
+ "requires a Windows test_stdio_server in the Wine-exec environment"
+ );
+ skip_if_no_network!(Ok(()));
+
+ const SCREENSHOT: &str = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGP4z8DwHwAFAAH/iZk9HQAAAABJRU5ErkJggg==";
+ let server = responses::start_mock_server().await;
+ let mcp_server_bin = remote_aware_stdio_server_bin()?;
+ let mut builder = test_codex().with_config(move |config| {
+ config
+ .features
+ .enable(Feature::CodeMode)
+ .expect("enable Code Mode");
+ let mcp = serde_json::from_value(serde_json::json!({
+ "command": mcp_server_bin,
+ "environment_id": remote_aware_environment_id(),
+ "env": { "MCP_TEST_ENABLE_NODE_REPL_JS": "1" },
+ "omit_tools_from": ["deferred"],
+ }))
+ .expect("valid node_repl MCP server config");
+ config
+ .mcp_servers
+ .set(HashMap::from([("node_repl".to_owned(), mcp)]))
+ .expect("configure node_repl MCP server");
+ });
+ let test = builder.build_with_auto_env(&server).await?;
+ core_test_support::wait_for_mcp_server(&test.codex, "node_repl").await?;
+ let evidence = test
+ .codex
+ .thread_extension_data()
+ .get_or_init(NodeReplReviewEvidence::default);
+ evidence.enable_image_capture();
+
+ let response_mock = responses::mount_sse_sequence(
+ &server,
+ vec![
+ sse(vec![
+ ev_custom_tool_call(
+ "node-repl-image",
+ "exec",
+ r#"for (let index = 0; index < 2; index++) await tools.mcp__node_repl__js({ code: 'await nodeRepl.emitImage(await tab.screenshot())' });"#,
+ ),
+ ev_completed("response-node-repl"),
+ ]),
+ sse(vec![ev_completed("response-done")]),
+ ],
+ )
+ .await;
+ test.submit_text_turn("capture a computer-use screenshot")
+ .await?;
+
+ assert_eq!(
+ evidence.images(),
+ vec![ContentItem::InputImage {
+ image_url: SCREENSHOT.to_owned(),
+ detail: Some(ImageDetail::Low),
+ }]
+ );
+ let parent_request = response_mock
+ .requests()
+ .pop()
+ .expect("parent turn should complete");
+ assert!(!serde_json::to_string(&parent_request.input())?.contains(SCREENSHOT));
+
+ Ok(())
+}
+
#[cfg_attr(windows, ignore = "no exec_command on Windows")]
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[test_case(false, false, false, None; "disabled")]
diff --git a/codex-rs/ext/guardian-v2/src/extension.rs b/codex-rs/ext/guardian-v2/src/extension.rs
index fffe9dcef5..eaa8fb6f6f 100644
--- a/codex-rs/ext/guardian-v2/src/extension.rs
+++ b/codex-rs/ext/guardian-v2/src/extension.rs
@@ -7,6 +7,7 @@ use std::time::SystemTime;
use codex_core::ThreadManager;
use codex_core::config::Config;
+use codex_core::context::NodeReplReviewEvidence;
use codex_extension_api::ApprovalReviewContributor;
use codex_extension_api::ExtensionData;
use codex_extension_api::ExtensionEventSink;
@@ -266,6 +267,12 @@ impl ThreadLifecycleContributor<Config> for GuardianV2Extension {
match sampler {
Ok(sampler) => {
+ if guardian_config.transcript.include_images {
+ input
+ .thread_store
+ .get_or_init(NodeReplReviewEvidence::default)
+ .enable_image_capture();
+ }
input.thread_store.insert(sampler);
input.thread_store.insert(guardian_config);
input
@@ -424,6 +431,15 @@ impl ToolLifecycleContributor for GuardianV2Extension {
.as_ref()
.and_then(|model| model.auto_review_model_override.clone());
let conversation_history = Arc::clone(&input.conversation_history);
+ let node_repl_images = if guardian_config.transcript.include_images {
+ input
+ .thread_store
+ .get::<NodeReplReviewEvidence>()
+ .map(|evidence| evidence.images())
+ .unwrap_or_default()
+ } else {
+ Vec::new()
+ };
tokio::spawn(async move {
let transcript = guardian_config
@@ -431,7 +447,7 @@ impl ToolLifecycleContributor for GuardianV2Extension {
.build(conversation_history.items());
let images = guardian_config
.transcript
- .images(conversation_history.items());
+ .images(conversation_history.items(), node_repl_images);
drop(conversation_history);
let planned_action = match action.render(guardian_config.max_action_tokens) {
Ok(planned_action) => planned_action,
diff --git a/codex-rs/ext/guardian-v2/src/transcript.rs b/codex-rs/ext/guardian-v2/src/transcript.rs
index 10f9ac2f8b..bae1bb6049 100644
--- a/codex-rs/ext/guardian-v2/src/transcript.rs
+++ b/codex-rs/ext/guardian-v2/src/transcript.rs
@@ -63,6 +63,7 @@ impl TranscriptConfig {
pub(crate) fn images<'a>(
&self,
items: impl IntoIterator<Item = &'a ResponseItem>,
+ node_repl_images: impl IntoIterator<Item = ContentItem>,
) -> Vec<ContentItem> {
if !self.include_images {
return Vec::new();
@@ -117,6 +118,13 @@ impl TranscriptConfig {
_ => {}
}
}
+ if self.sources.contains(&TranscriptSource::ToolOutputs) {
+ for image in node_repl_images {
+ if let ContentItem::InputImage { image_url, detail } = image {
+ include_image(&image_url, detail);
+ }
+ }
+ }
images.into_iter().collect()
}