Lightweight coding agent that runs in your terminal
Rust
Prevent later Guardian scores from releasing earlier pending reviews (#51400)
jifa4ebc509f4cd
diff --git a/codex-rs/app-server/tests/suite/v2/guardian_pending_score_tests.rs b/codex-rs/app-server/tests/suite/v2/guardian_pending_score_tests.rsnew file mode 100644index 0000000000..58f15c9829--- /dev/null+++ b/codex-rs/app-server/tests/suite/v2/guardian_pending_score_tests.rs@@ -0,0 +1,159 @@+//! A later LOW releases its own review without releasing an earlier pending review.++use super::*;+use codex_app_server_protocol::GuardianApprovalReviewStatus;+use codex_app_server_protocol::ItemGuardianApprovalReviewCompletedNotification;+use futures::StreamExt;+use pretty_assertions::assert_eq;++#[tokio::test(flavor = "multi_thread", worker_threads = 2)]+async fn later_low_does_not_release_earlier_review() -> Result<()> {+ skip_if_no_network!(Ok(()));+ let state = Arc::new(MockResponsesState {+ luna_gates: (0..2).map(|_| Notify::new()).collect(),+ gate_each_guardian_review: true,+ review_outcome: ReviewOutcome::Deny,+ ..Default::default()+ });+ let listener = TcpListener::bind("127.0.0.1:0").await?;+ let responses_url = format!("http://{}", listener.local_addr()?);+ let router = Router::new()+ .route(+ "/v1/responses",+ get(luna_websocket).post(+ |State(state): State<Arc<MockResponsesState>>, Json(request): Json<Value>| async move {+ if request.pointer("/client_metadata/x-openai-subagent") == Some(&json!("guardian")) {+ return parent_response(State(state), Json(request)).await.into_response();+ }+ if state.parent_requests.fetch_add(/*val*/ 1, Ordering::SeqCst) > 0 {+ return (+ [(header::CONTENT_TYPE, "text/event-stream")],+ responses::sse(vec![+ responses::ev_assistant_message("done", "done"),+ responses::ev_completed("done"),+ ]),+ ).into_response();+ }+ let stream = futures::stream::iter(0..=2).then(move |index| {+ let state = Arc::clone(&state);+ async move {+ // Start each action only once the previous action is awaiting+ // synchronous review and its classifier request has arrived.+ if index > 0 {+ wait_for_luna_request(&state, index - 1).await.unwrap();+ wait_for_guardian_reviews(&state, index).await.unwrap();+ }+ let mut events = Vec::new();+ if index == 0 {+ events.push(responses::ev_response_created("actions"));+ }+ if index < 2 {+ events.push(responses::ev_function_call_with_namespace(+ &format!("action-{index}"),+ &format!("mcp__{TEST_SERVER_NAME}"),+ TEST_TOOL_NAME,+ &json!({"message": format!("action-{index}")}).to_string(),+ ));+ } else {+ events.push(responses::ev_completed("actions"));+ }+ Ok::<_, std::convert::Infallible>(responses::sse(events))+ }+ });+ (+ [(header::CONTENT_TYPE, "text/event-stream")],+ axum::body::Body::from_stream(stream),+ ).into_response()+ },+ ),+ )+ .with_state(Arc::clone(&state));+ let responses_server = tokio::spawn(async move {+ let _ = axum::serve(listener, router).await;+ });+ let (mcp_url, mcp_server) = start_mcp_server(/*sensitive_action*/ None).await?;+ let codex_home = TempDir::new()?;+ MockResponsesConfig::new(&responses_url)+ .with_model(MODEL)+ .with_provider_config("supports_websockets = false")+ .with_approval_policy("on-request")+ .with_root_config("approvals_reviewer = \"auto_review\"")+ .with_extra_config(&format!(+ "[mcp_servers.{TEST_SERVER_NAME}]\nurl = \"{mcp_url}/mcp\"\ndefault_tools_approval_mode = \"prompt\"\nsupports_parallel_tool_calls = true\n\n[features.guardianv2]\nenabled = true\nmax_tool_call_lag = 0\n\n[features.guardianv2.review_scope]\ncomputer_use_only = false"+ ))+ .enable_feature(Feature::GuardianApproval)+ .write(codex_home.path())?;+ let mut app_server = TestAppServer::builder()+ .with_codex_home(codex_home.path())+ .build_initialized_with_timeout(TIMEOUT)+ .await?;+ let thread = app_server+ .start_thread(ThreadStartParams::default())+ .await?+ .thread;+ let id = app_server+ .send_turn_start_request(TurnStartParams {+ thread_id: thread.id,+ input: vec![UserInput::Text {+ text: USER_CONTEXT.to_owned(),+ text_elements: Vec::new(),+ }],+ ..Default::default()+ })+ .await?;+ let _: TurnStartResponse = timeout(TIMEOUT, app_server.read_response(id)).await??;+ wait_for_luna_request(&state, /*index*/ 1).await?;+ wait_for_guardian_reviews(&state, /*expected*/ 2).await?;++ // Both reviews are pending. Only the later action gets a LOW, at thread lag zero.+ state.luna_gates[1].notify_one();+ let approved: ItemGuardianApprovalReviewCompletedNotification = timeout(+ TIMEOUT,+ app_server.read_notification("item/autoApprovalReview/completed"),+ )+ .await??;+ assert_eq!(+ (+ approved.target_item_id,+ approved.review.status,+ approved.review.risk_level+ ),+ (+ Some("action-1".to_owned()),+ GuardianApprovalReviewStatus::Approved,+ None+ )+ );+ assert!(+ timeout(+ Duration::from_millis(/*millis*/ 250),+ app_server.read_notification::<ItemGuardianApprovalReviewCompletedNotification>(+ "item/autoApprovalReview/completed"+ ),+ )+ .await+ .is_err()+ );++ // The earlier action must still wait for, and honor, its synchronous denial.+ state.allow_guardian_review.notify_waiters();+ let denied: ItemGuardianApprovalReviewCompletedNotification = timeout(+ TIMEOUT,+ app_server.read_notification("item/autoApprovalReview/completed"),+ )+ .await??;+ assert_eq!(+ (denied.target_item_id, denied.review.status),+ (+ Some("action-0".to_owned()),+ GuardianApprovalReviewStatus::Denied+ )+ );+ let completed: TurnCompletedNotification =+ timeout(TIMEOUT, app_server.read_notification("turn/completed")).await??;+ assert_eq!(completed.turn.status, TurnStatus::Completed);+ app_server.shutdown_gracefully().await?;+ mcp_server.abort();+ responses_server.abort();+ Ok(())+}diff --git a/codex-rs/app-server/tests/suite/v2/guardian_v2.rs b/codex-rs/app-server/tests/suite/v2/guardian_v2.rsindex 2d3b581daf..dd9017e9a9 100644--- a/codex-rs/app-server/tests/suite/v2/guardian_v2.rs+++ b/codex-rs/app-server/tests/suite/v2/guardian_v2.rs@@ -96,6 +96,9 @@ mod code_mode;#[path = "guardian_action_budget_tests.rs"]mod action_budget;+#[path = "guardian_pending_score_tests.rs"]+mod pending_scores;+const TIMEOUT: Duration = Duration::from_secs(30);const MODEL: &str = "mock-model";const REQUIRED_MODEL: &str = "protected-model";@@ -182,6 +185,7 @@ struct MockResponsesState {luna_completions: AtomicUsize,root_thread_id: Mutex<Option<String>>,allow_luna: Notify,+ luna_gates: Vec<Notify>,allow_guardian_review: Notify,gate_each_guardian_review: bool,gate_second_tool_on_classification: bool,@@ -615,12 +619,18 @@ async fn luna_response(state: &MockResponsesState, request: Value) -> Vec<Value>request["prompt_cache_key"] == format!("guardian-v2:{thread_id}")});if !is_root_sample {+ let index = {+ let mut requests = state.luna_requests.lock().expect("Luna request lock");+ let index = requests.len();+ requests.push(request);+ index+ };state- .luna_requests- .lock()- .expect("Luna request lock should not be poisoned")- .push(request);- state.allow_luna.notified().await;+ .luna_gates+ .get(index)+ .unwrap_or(&state.allow_luna)+ .notified()+ .await;}let classification = if state.invalid_classification {"invalid"@@ -2374,23 +2384,16 @@ async fn guardian_v2_computer_use_only_scopes_classification_and_fast_reviews(.await}-enum ApprovalCompletion {- Sync,- AsyncLow,-}--#[test_case("cua_repl", &["js", "js"], ApprovalCompletion::AsyncLow; "late low cancels review")]-#[test_case("node_repl", &["js", "js"], ApprovalCompletion::Sync; "browser startup")]-#[test_case("cua_repl", &["js", "js"], ApprovalCompletion::Sync; "computer use startup")]-#[test_case("node_repl", &["js_reset", "js", "js"], ApprovalCompletion::Sync; "browser reset before execution")]-#[test_case("cua_repl", &["js_reset", "js", "js"], ApprovalCompletion::Sync; "computer use reset before execution")]-#[test_case("node_repl", &["js_add_node_module_dir", "js", "js"], ApprovalCompletion::Sync; "browser setup before execution")]-#[test_case("cua_repl", &["js_add_node_module_dir", "js", "js"], ApprovalCompletion::Sync; "computer use setup before execution")]+#[test_case("node_repl", &["js", "js"]; "browser startup")]+#[test_case("cua_repl", &["js", "js"]; "computer use startup")]+#[test_case("node_repl", &["js_reset", "js", "js"]; "browser reset before execution")]+#[test_case("cua_repl", &["js_reset", "js", "js"]; "computer use reset before execution")]+#[test_case("node_repl", &["js_add_node_module_dir", "js", "js"]; "browser setup before execution")]+#[test_case("cua_repl", &["js_add_node_module_dir", "js", "js"]; "computer use setup before execution")]#[tokio::test(flavor = "multi_thread", worker_threads = 2)]async fn first_cua_review_does_not_wait_for_initial_score(server_name: &'static str,tool_sequence: &'static [&'static str],- completion: ApprovalCompletion,) -> Result<()> {skip_if_no_network!(Ok(()));let state = Arc::new(MockResponsesState {@@ -2467,24 +2470,7 @@ async fn first_cua_review_does_not_wait_for_initial_score(state.parent_requests.load(Ordering::SeqCst),tool_sequence.len());- if !matches!(completion, ApprovalCompletion::Sync) {- state.allow_luna.notify_one();- let review: codex_app_server_protocol::ItemGuardianApprovalReviewCompletedNotification =- timeout(- TIMEOUT,- app_server.read_notification("item/autoApprovalReview/completed"),- )- .await??;- assert_eq!(- (review.review.status, review.review.risk_level),- (- codex_app_server_protocol::GuardianApprovalReviewStatus::Approved,- None- )- );- } else {- state.allow_guardian_review.notify_one();- }+ state.allow_guardian_review.notify_one();let completed: TurnCompletedNotification =timeout(TIMEOUT, app_server.read_notification("turn/completed")).await??;assert_eq!(completed.thread_id, thread.id);diff --git a/codex-rs/ext/guardian-v2/src/async_scorer/approval.rs b/codex-rs/ext/guardian-v2/src/async_scorer/approval.rsindex c979adb7d2..4a4c78b174 100644--- a/codex-rs/ext/guardian-v2/src/async_scorer/approval.rs+++ b/codex-rs/ext/guardian-v2/src/async_scorer/approval.rs@@ -105,7 +105,7 @@ impl GuardianApprovalReviewer {match cached_evidence(thread, input, config, &policy, input.metrics.as_deref()).await{- Ok(()) => return ApprovalDecision::Allow,+ Ok(_) => return ApprovalDecision::Allow,Err(reason) => reason,}}@@ -133,9 +133,8 @@ impl GuardianApprovalReviewer {// Subscribe before checking so a concurrently published score is not lost.let mut updates = progress.updates.subscribe();loop {- if cached_evidence(thread, input, config, &policy, /*metrics*/ None)- .await- .is_ok()+ if let Ok(true) =+ cached_evidence(thread, input, config, &policy, /*metrics*/ None).await{return;}@@ -157,13 +156,14 @@ impl GuardianApprovalReviewer {}}+/// Checks score reuse; the success value says whether it can also release a pending review.async fn cached_evidence(thread: &CodexThread,input: &ApprovalDecisionInput<'_>,config: &GuardianV2Config,policy: &GuardianModelPolicy,metrics: Option<&dyn ExtensionMetrics>,-) -> Result<(), GuardianReviewReason> {+) -> Result<bool, GuardianReviewReason> {let store = input.thread_store;let Some(progress) = store.get::<GuardianV2ScoreProgress>() else {record_fast_decision(metrics, "deferred", "missing_score");@@ -217,7 +217,7 @@ async fn cached_evidence(&& cached.js_executions == 1{record_fast_decision(metrics, "approved", "initial_cua_call");- return Ok(());+ return Ok(false);}let Some(permissions) = input.permissions else {record_fast_decision(metrics, "deferred", "permission_resolution_error");@@ -268,7 +268,7 @@ async fn cached_evidence()} else {record_fast_decision(metrics, "approved", "low_risk");- return Ok(());+ return Ok(cached.score_at_or_before_action);}}Some(score) if score >= config.review_threshold => {diff --git a/codex-rs/ext/guardian-v2/src/async_scorer/score.rs b/codex-rs/ext/guardian-v2/src/async_scorer/score.rsindex 99acdea8b7..e732ad0812 100644--- a/codex-rs/ext/guardian-v2/src/async_scorer/score.rs+++ b/codex-rs/ext/guardian-v2/src/async_scorer/score.rs@@ -25,6 +25,7 @@ pub(super) struct GuardianV2ScoreProgress {#[derive(Default)]struct ScoreState {score: Option<SecurityRiskScore>,+ score_index: usize,wrapper_lag: WrapperLag,latest_tool_call: usize,js_executions: usize,@@ -38,6 +39,7 @@ struct ScoreState {#[derive(Debug, PartialEq)]pub(super) struct CachedScore {pub(super) lag: usize,+ pub(super) score_at_or_before_action: bool,pub(super) has_unscored_failure: bool,pub(super) js_executions: usize,pub(super) oversized: bool,@@ -115,6 +117,7 @@ impl GuardianV2ScoreProgress {.is_none_or(|previous| previous.sampled_at < score.sampled_at);if accepted {state.score = Some(score);+ state.score_index = index;state.authorization = Some(authorization);state.latest_scored_tool_call = state.latest_scored_tool_call.max(index);self.updates.send_replace(());@@ -156,6 +159,11 @@ impl GuardianV2ScoreProgress {.lock().unwrap_or_else(std::sync::PoisonError::into_inner);CachedScore {+ // Use the accepted score's source, not the coverage high-water mark.+ score_at_or_before_action: state+ .wrapper_lag+ .get(call_id)+ .is_some_and(|start| state.score_index <= start.index),lag: state.latest_tool_call.saturating_sub(state.latest_scored_tool_call)diff --git a/codex-rs/ext/guardian-v2/src/async_scorer/wrapper_lag.rs b/codex-rs/ext/guardian-v2/src/async_scorer/wrapper_lag.rsindex b4e198b4f6..44566c9371 100644--- a/codex-rs/ext/guardian-v2/src/async_scorer/wrapper_lag.rs+++ b/codex-rs/ext/guardian-v2/src/async_scorer/wrapper_lag.rs@@ -1,5 +1,5 @@-//! Discounts only an approval's own unscored code-mode wrapper. Missing provenance-//! keeps the full lag; the bounded history never changes scoring or failure order.+//! Tracks action order and discounts an approval's own unscored code-mode wrapper.+//! Missing provenance keeps the full lag; bounded history never changes scoring or failure order.//! The score owner's state lock protects this history.use std::collections::VecDeque;@@ -9,9 +9,9 @@ use codex_extension_api::ToolPayload;use codex_extension_api::ToolStartInput;use codex_protocol::ResponseItemId;-struct ToolStart {+pub(super) struct ToolStart {call_id: String,- index: usize,+ pub(super) index: usize,wrapper_item_id: Option<ResponseItemId>,parent_wrapper_index: Option<usize>,}@@ -54,16 +54,18 @@ impl WrapperLag {});}+ pub(super) fn get(&self, call_id: Option<&str>) -> Option<&ToolStart> {+ let call_id = call_id?;+ self.starts+ .iter()+ .rev()+ .find(|start| start.call_id == call_id)+ }+pub(super) fn discount(&self, call_id: Option<&str>, latest_scored: usize) -> usize {usize::from(- call_id- .and_then(|call_id| {- self.starts- .iter()- .rev()- .find(|start| start.call_id == call_id)- .and_then(|start| start.parent_wrapper_index)- })+ self.get(call_id)+ .and_then(|start| start.parent_wrapper_index).is_some_and(|parent| parent > latest_scored),)}