Skip to content

armgit / codex

Public Imported from GitHub
0

Lightweight coding agent that runs in your terminal

Rust

Prevent later Guardian scores from releasing earlier pending reviews (#51400)

jifa4ebc509f4cd
diff --git a/codex-rs/app-server/tests/suite/v2/guardian_pending_score_tests.rs b/codex-rs/app-server/tests/suite/v2/guardian_pending_score_tests.rs
new file mode 100644
index 0000000000..58f15c9829
--- /dev/null
+++ b/codex-rs/app-server/tests/suite/v2/guardian_pending_score_tests.rs
@@ -0,0 +1,159 @@
+//! A later LOW releases its own review without releasing an earlier pending review.
+
+use super::*;
+use codex_app_server_protocol::GuardianApprovalReviewStatus;
+use codex_app_server_protocol::ItemGuardianApprovalReviewCompletedNotification;
+use futures::StreamExt;
+use pretty_assertions::assert_eq;
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn later_low_does_not_release_earlier_review() -> Result<()> {
+ skip_if_no_network!(Ok(()));
+ let state = Arc::new(MockResponsesState {
+ luna_gates: (0..2).map(|_| Notify::new()).collect(),
+ gate_each_guardian_review: true,
+ review_outcome: ReviewOutcome::Deny,
+ ..Default::default()
+ });
+ let listener = TcpListener::bind("127.0.0.1:0").await?;
+ let responses_url = format!("http://{}", listener.local_addr()?);
+ let router = Router::new()
+ .route(
+ "/v1/responses",
+ get(luna_websocket).post(
+ |State(state): State<Arc<MockResponsesState>>, Json(request): Json<Value>| async move {
+ if request.pointer("/client_metadata/x-openai-subagent") == Some(&json!("guardian")) {
+ return parent_response(State(state), Json(request)).await.into_response();
+ }
+ if state.parent_requests.fetch_add(/*val*/ 1, Ordering::SeqCst) > 0 {
+ return (
+ [(header::CONTENT_TYPE, "text/event-stream")],
+ responses::sse(vec![
+ responses::ev_assistant_message("done", "done"),
+ responses::ev_completed("done"),
+ ]),
+ ).into_response();
+ }
+ let stream = futures::stream::iter(0..=2).then(move |index| {
+ let state = Arc::clone(&state);
+ async move {
+ // Start each action only once the previous action is awaiting
+ // synchronous review and its classifier request has arrived.
+ if index > 0 {
+ wait_for_luna_request(&state, index - 1).await.unwrap();
+ wait_for_guardian_reviews(&state, index).await.unwrap();
+ }
+ let mut events = Vec::new();
+ if index == 0 {
+ events.push(responses::ev_response_created("actions"));
+ }
+ if index < 2 {
+ events.push(responses::ev_function_call_with_namespace(
+ &format!("action-{index}"),
+ &format!("mcp__{TEST_SERVER_NAME}"),
+ TEST_TOOL_NAME,
+ &json!({"message": format!("action-{index}")}).to_string(),
+ ));
+ } else {
+ events.push(responses::ev_completed("actions"));
+ }
+ Ok::<_, std::convert::Infallible>(responses::sse(events))
+ }
+ });
+ (
+ [(header::CONTENT_TYPE, "text/event-stream")],
+ axum::body::Body::from_stream(stream),
+ ).into_response()
+ },
+ ),
+ )
+ .with_state(Arc::clone(&state));
+ let responses_server = tokio::spawn(async move {
+ let _ = axum::serve(listener, router).await;
+ });
+ let (mcp_url, mcp_server) = start_mcp_server(/*sensitive_action*/ None).await?;
+ let codex_home = TempDir::new()?;
+ MockResponsesConfig::new(&responses_url)
+ .with_model(MODEL)
+ .with_provider_config("supports_websockets = false")
+ .with_approval_policy("on-request")
+ .with_root_config("approvals_reviewer = \"auto_review\"")
+ .with_extra_config(&format!(
+ "[mcp_servers.{TEST_SERVER_NAME}]\nurl = \"{mcp_url}/mcp\"\ndefault_tools_approval_mode = \"prompt\"\nsupports_parallel_tool_calls = true\n\n[features.guardianv2]\nenabled = true\nmax_tool_call_lag = 0\n\n[features.guardianv2.review_scope]\ncomputer_use_only = false"
+ ))
+ .enable_feature(Feature::GuardianApproval)
+ .write(codex_home.path())?;
+ let mut app_server = TestAppServer::builder()
+ .with_codex_home(codex_home.path())
+ .build_initialized_with_timeout(TIMEOUT)
+ .await?;
+ let thread = app_server
+ .start_thread(ThreadStartParams::default())
+ .await?
+ .thread;
+ let id = app_server
+ .send_turn_start_request(TurnStartParams {
+ thread_id: thread.id,
+ input: vec![UserInput::Text {
+ text: USER_CONTEXT.to_owned(),
+ text_elements: Vec::new(),
+ }],
+ ..Default::default()
+ })
+ .await?;
+ let _: TurnStartResponse = timeout(TIMEOUT, app_server.read_response(id)).await??;
+ wait_for_luna_request(&state, /*index*/ 1).await?;
+ wait_for_guardian_reviews(&state, /*expected*/ 2).await?;
+
+ // Both reviews are pending. Only the later action gets a LOW, at thread lag zero.
+ state.luna_gates[1].notify_one();
+ let approved: ItemGuardianApprovalReviewCompletedNotification = timeout(
+ TIMEOUT,
+ app_server.read_notification("item/autoApprovalReview/completed"),
+ )
+ .await??;
+ assert_eq!(
+ (
+ approved.target_item_id,
+ approved.review.status,
+ approved.review.risk_level
+ ),
+ (
+ Some("action-1".to_owned()),
+ GuardianApprovalReviewStatus::Approved,
+ None
+ )
+ );
+ assert!(
+ timeout(
+ Duration::from_millis(/*millis*/ 250),
+ app_server.read_notification::<ItemGuardianApprovalReviewCompletedNotification>(
+ "item/autoApprovalReview/completed"
+ ),
+ )
+ .await
+ .is_err()
+ );
+
+ // The earlier action must still wait for, and honor, its synchronous denial.
+ state.allow_guardian_review.notify_waiters();
+ let denied: ItemGuardianApprovalReviewCompletedNotification = timeout(
+ TIMEOUT,
+ app_server.read_notification("item/autoApprovalReview/completed"),
+ )
+ .await??;
+ assert_eq!(
+ (denied.target_item_id, denied.review.status),
+ (
+ Some("action-0".to_owned()),
+ GuardianApprovalReviewStatus::Denied
+ )
+ );
+ let completed: TurnCompletedNotification =
+ timeout(TIMEOUT, app_server.read_notification("turn/completed")).await??;
+ assert_eq!(completed.turn.status, TurnStatus::Completed);
+ app_server.shutdown_gracefully().await?;
+ mcp_server.abort();
+ responses_server.abort();
+ Ok(())
+}
diff --git a/codex-rs/app-server/tests/suite/v2/guardian_v2.rs b/codex-rs/app-server/tests/suite/v2/guardian_v2.rs
index 2d3b581daf..dd9017e9a9 100644
--- a/codex-rs/app-server/tests/suite/v2/guardian_v2.rs
+++ b/codex-rs/app-server/tests/suite/v2/guardian_v2.rs
@@ -96,6 +96,9 @@ mod code_mode;
#[path = "guardian_action_budget_tests.rs"]
mod action_budget;
+#[path = "guardian_pending_score_tests.rs"]
+mod pending_scores;
+
const TIMEOUT: Duration = Duration::from_secs(30);
const MODEL: &str = "mock-model";
const REQUIRED_MODEL: &str = "protected-model";
@@ -182,6 +185,7 @@ struct MockResponsesState {
luna_completions: AtomicUsize,
root_thread_id: Mutex<Option<String>>,
allow_luna: Notify,
+ luna_gates: Vec<Notify>,
allow_guardian_review: Notify,
gate_each_guardian_review: bool,
gate_second_tool_on_classification: bool,
@@ -615,12 +619,18 @@ async fn luna_response(state: &MockResponsesState, request: Value) -> Vec<Value>
request["prompt_cache_key"] == format!("guardian-v2:{thread_id}")
});
if !is_root_sample {
+ let index = {
+ let mut requests = state.luna_requests.lock().expect("Luna request lock");
+ let index = requests.len();
+ requests.push(request);
+ index
+ };
state
- .luna_requests
- .lock()
- .expect("Luna request lock should not be poisoned")
- .push(request);
- state.allow_luna.notified().await;
+ .luna_gates
+ .get(index)
+ .unwrap_or(&state.allow_luna)
+ .notified()
+ .await;
}
let classification = if state.invalid_classification {
"invalid"
@@ -2374,23 +2384,16 @@ async fn guardian_v2_computer_use_only_scopes_classification_and_fast_reviews(
.await
}
-enum ApprovalCompletion {
- Sync,
- AsyncLow,
-}
-
-#[test_case("cua_repl", &["js", "js"], ApprovalCompletion::AsyncLow; "late low cancels review")]
-#[test_case("node_repl", &["js", "js"], ApprovalCompletion::Sync; "browser startup")]
-#[test_case("cua_repl", &["js", "js"], ApprovalCompletion::Sync; "computer use startup")]
-#[test_case("node_repl", &["js_reset", "js", "js"], ApprovalCompletion::Sync; "browser reset before execution")]
-#[test_case("cua_repl", &["js_reset", "js", "js"], ApprovalCompletion::Sync; "computer use reset before execution")]
-#[test_case("node_repl", &["js_add_node_module_dir", "js", "js"], ApprovalCompletion::Sync; "browser setup before execution")]
-#[test_case("cua_repl", &["js_add_node_module_dir", "js", "js"], ApprovalCompletion::Sync; "computer use setup before execution")]
+#[test_case("node_repl", &["js", "js"]; "browser startup")]
+#[test_case("cua_repl", &["js", "js"]; "computer use startup")]
+#[test_case("node_repl", &["js_reset", "js", "js"]; "browser reset before execution")]
+#[test_case("cua_repl", &["js_reset", "js", "js"]; "computer use reset before execution")]
+#[test_case("node_repl", &["js_add_node_module_dir", "js", "js"]; "browser setup before execution")]
+#[test_case("cua_repl", &["js_add_node_module_dir", "js", "js"]; "computer use setup before execution")]
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn first_cua_review_does_not_wait_for_initial_score(
server_name: &'static str,
tool_sequence: &'static [&'static str],
- completion: ApprovalCompletion,
) -> Result<()> {
skip_if_no_network!(Ok(()));
let state = Arc::new(MockResponsesState {
@@ -2467,24 +2470,7 @@ async fn first_cua_review_does_not_wait_for_initial_score(
state.parent_requests.load(Ordering::SeqCst),
tool_sequence.len()
);
- if !matches!(completion, ApprovalCompletion::Sync) {
- state.allow_luna.notify_one();
- let review: codex_app_server_protocol::ItemGuardianApprovalReviewCompletedNotification =
- timeout(
- TIMEOUT,
- app_server.read_notification("item/autoApprovalReview/completed"),
- )
- .await??;
- assert_eq!(
- (review.review.status, review.review.risk_level),
- (
- codex_app_server_protocol::GuardianApprovalReviewStatus::Approved,
- None
- )
- );
- } else {
- state.allow_guardian_review.notify_one();
- }
+ state.allow_guardian_review.notify_one();
let completed: TurnCompletedNotification =
timeout(TIMEOUT, app_server.read_notification("turn/completed")).await??;
assert_eq!(completed.thread_id, thread.id);
diff --git a/codex-rs/ext/guardian-v2/src/async_scorer/approval.rs b/codex-rs/ext/guardian-v2/src/async_scorer/approval.rs
index c979adb7d2..4a4c78b174 100644
--- a/codex-rs/ext/guardian-v2/src/async_scorer/approval.rs
+++ b/codex-rs/ext/guardian-v2/src/async_scorer/approval.rs
@@ -105,7 +105,7 @@ impl GuardianApprovalReviewer {
match cached_evidence(thread, input, config, &policy, input.metrics.as_deref())
.await
{
- Ok(()) => return ApprovalDecision::Allow,
+ Ok(_) => return ApprovalDecision::Allow,
Err(reason) => reason,
}
}
@@ -133,9 +133,8 @@ impl GuardianApprovalReviewer {
// Subscribe before checking so a concurrently published score is not lost.
let mut updates = progress.updates.subscribe();
loop {
- if cached_evidence(thread, input, config, &policy, /*metrics*/ None)
- .await
- .is_ok()
+ if let Ok(true) =
+ cached_evidence(thread, input, config, &policy, /*metrics*/ None).await
{
return;
}
@@ -157,13 +156,14 @@ impl GuardianApprovalReviewer {
}
}
+/// Checks score reuse; the success value says whether it can also release a pending review.
async fn cached_evidence(
thread: &CodexThread,
input: &ApprovalDecisionInput<'_>,
config: &GuardianV2Config,
policy: &GuardianModelPolicy,
metrics: Option<&dyn ExtensionMetrics>,
-) -> Result<(), GuardianReviewReason> {
+) -> Result<bool, GuardianReviewReason> {
let store = input.thread_store;
let Some(progress) = store.get::<GuardianV2ScoreProgress>() else {
record_fast_decision(metrics, "deferred", "missing_score");
@@ -217,7 +217,7 @@ async fn cached_evidence(
&& cached.js_executions == 1
{
record_fast_decision(metrics, "approved", "initial_cua_call");
- return Ok(());
+ return Ok(false);
}
let Some(permissions) = input.permissions else {
record_fast_decision(metrics, "deferred", "permission_resolution_error");
@@ -268,7 +268,7 @@ async fn cached_evidence(
)
} else {
record_fast_decision(metrics, "approved", "low_risk");
- return Ok(());
+ return Ok(cached.score_at_or_before_action);
}
}
Some(score) if score >= config.review_threshold => {
diff --git a/codex-rs/ext/guardian-v2/src/async_scorer/score.rs b/codex-rs/ext/guardian-v2/src/async_scorer/score.rs
index 99acdea8b7..e732ad0812 100644
--- a/codex-rs/ext/guardian-v2/src/async_scorer/score.rs
+++ b/codex-rs/ext/guardian-v2/src/async_scorer/score.rs
@@ -25,6 +25,7 @@ pub(super) struct GuardianV2ScoreProgress {
#[derive(Default)]
struct ScoreState {
score: Option<SecurityRiskScore>,
+ score_index: usize,
wrapper_lag: WrapperLag,
latest_tool_call: usize,
js_executions: usize,
@@ -38,6 +39,7 @@ struct ScoreState {
#[derive(Debug, PartialEq)]
pub(super) struct CachedScore {
pub(super) lag: usize,
+ pub(super) score_at_or_before_action: bool,
pub(super) has_unscored_failure: bool,
pub(super) js_executions: usize,
pub(super) oversized: bool,
@@ -115,6 +117,7 @@ impl GuardianV2ScoreProgress {
.is_none_or(|previous| previous.sampled_at < score.sampled_at);
if accepted {
state.score = Some(score);
+ state.score_index = index;
state.authorization = Some(authorization);
state.latest_scored_tool_call = state.latest_scored_tool_call.max(index);
self.updates.send_replace(());
@@ -156,6 +159,11 @@ impl GuardianV2ScoreProgress {
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
CachedScore {
+ // Use the accepted score's source, not the coverage high-water mark.
+ score_at_or_before_action: state
+ .wrapper_lag
+ .get(call_id)
+ .is_some_and(|start| state.score_index <= start.index),
lag: state
.latest_tool_call
.saturating_sub(state.latest_scored_tool_call)
diff --git a/codex-rs/ext/guardian-v2/src/async_scorer/wrapper_lag.rs b/codex-rs/ext/guardian-v2/src/async_scorer/wrapper_lag.rs
index b4e198b4f6..44566c9371 100644
--- a/codex-rs/ext/guardian-v2/src/async_scorer/wrapper_lag.rs
+++ b/codex-rs/ext/guardian-v2/src/async_scorer/wrapper_lag.rs
@@ -1,5 +1,5 @@
-//! Discounts only an approval's own unscored code-mode wrapper. Missing provenance
-//! keeps the full lag; the bounded history never changes scoring or failure order.
+//! Tracks action order and discounts an approval's own unscored code-mode wrapper.
+//! Missing provenance keeps the full lag; bounded history never changes scoring or failure order.
//! The score owner's state lock protects this history.
use std::collections::VecDeque;
@@ -9,9 +9,9 @@ use codex_extension_api::ToolPayload;
use codex_extension_api::ToolStartInput;
use codex_protocol::ResponseItemId;
-struct ToolStart {
+pub(super) struct ToolStart {
call_id: String,
- index: usize,
+ pub(super) index: usize,
wrapper_item_id: Option<ResponseItemId>,
parent_wrapper_index: Option<usize>,
}
@@ -54,16 +54,18 @@ impl WrapperLag {
});
}
+ pub(super) fn get(&self, call_id: Option<&str>) -> Option<&ToolStart> {
+ let call_id = call_id?;
+ self.starts
+ .iter()
+ .rev()
+ .find(|start| start.call_id == call_id)
+ }
+
pub(super) fn discount(&self, call_id: Option<&str>, latest_scored: usize) -> usize {
usize::from(
- call_id
- .and_then(|call_id| {
- self.starts
- .iter()
- .rev()
- .find(|start| start.call_id == call_id)
- .and_then(|start| start.parent_wrapper_index)
- })
+ self.get(call_id)
+ .and_then(|start| start.parent_wrapper_index)
.is_some_and(|parent| parent > latest_scored),
)
}