Lightweight coding agent that runs in your terminal
Rust
Add Guardian V2 decision metrics (#41100)
jife9a446d79dc2
diff --git a/codex-rs/ext/guardian-v2/src/async_scorer/extension.rs b/codex-rs/ext/guardian-v2/src/async_scorer/extension.rsindex d1e7202770..e3b1277615 100644--- a/codex-rs/ext/guardian-v2/src/async_scorer/extension.rs+++ b/codex-rs/ext/guardian-v2/src/async_scorer/extension.rs@@ -3,7 +3,6 @@ use std::sync::Arc;use std::sync::Weak;use std::sync::atomic::AtomicUsize;use std::sync::atomic::Ordering;-use std::time::Duration;use std::time::Instant;use std::time::SystemTime;@@ -46,6 +45,11 @@ use serde_json::json;use super::config::GuardianV2Config;use super::config::GuardianV2ReviewScope;+use super::metrics::REVIEW_FALLBACK_METRIC;+use super::metrics::TOOL_CALL_LAG_METRIC;+use super::metrics::record_classification;+use super::metrics::record_classification_risk;+use super::metrics::record_fast_decision;use super::review_evidence::render_review_evidence;use super::sampler::LunaSampler;use super::sampler::LunaSamplerConfig;@@ -254,12 +258,6 @@ pub enum StrictReviewReason {struct GuardianV2Enabled;-// Sampled when the approval contributor evaluates the cached Luna score.-const TOOL_CALL_LAG_METRIC: &str = "codex.guardian_v2.tool_call_lag";-const REVIEW_FALLBACK_METRIC: &str = "codex.guardian_v2.review_fallback";-const CLASSIFICATION_METRIC: &str = "codex.guardian_v2.classification";-const CLASSIFICATION_DURATION_METRIC: &str = "codex.guardian_v2.classification.duration_ms";-#[derive(Default)]struct GuardianV2ScoreProgress {latest_tool_call: AtomicUsize,@@ -268,23 +266,6 @@ struct GuardianV2ScoreProgress {metrics: Option<Arc<dyn ExtensionMetrics>>,}-fn record_classification(- metrics: Option<&dyn ExtensionMetrics>,- duration: Duration,- outcome: &str,-) {- let Some(metrics) = metrics else {- return;- };- let tags = [("outcome", outcome)];- metrics.counter(CLASSIFICATION_METRIC, /*inc*/ 1, &tags);- metrics.histogram(- CLASSIFICATION_DURATION_METRIC,- i64::try_from(duration.as_millis()).unwrap_or(i64::MAX),- &tags,- );-}-#[derive(Clone)]struct GuardianV2Extension {auth_manager: Arc<AuthManager>,@@ -413,17 +394,24 @@ impl ApprovalReviewContributor for GuardianV2Extension {thread_store.get::<GuardianV2Enabled>()?;let guardian_config = thread_store.get::<GuardianV2Config>()?;if guardian_config.review_scope == GuardianV2ReviewScope::ComputerUseOnly {- let action = serde_json::from_str::<serde_json::Value>(prompt).ok()?;+ let Ok(action) = serde_json::from_str::<serde_json::Value>(prompt) else {+ record_fast_decision(extension_metrics.as_deref(), "deferred", "out_of_scope");+ return None;+ };if action.get("tool").and_then(serde_json::Value::as_str) != Some("mcp_tool_call")|| !action.get("server").and_then(serde_json::Value::as_str).is_some_and(is_node_repl_backed_server){+ record_fast_decision(extension_metrics.as_deref(), "deferred", "out_of_scope");return None;}}- let score_progress = thread_store.get::<GuardianV2ScoreProgress>()?;+ let Some(score_progress) = thread_store.get::<GuardianV2ScoreProgress>() else {+ record_fast_decision(extension_metrics.as_deref(), "deferred", "missing_score");+ return None;+ };let latest_scored_tool_call = score_progress.latest_scored_tool_call.load(Ordering::Acquire);@@ -447,6 +435,7 @@ impl ApprovalReviewContributor for GuardianV2Extension {&[("fallback_reason", "score_lag")],);}+ record_fast_decision(extension_metrics.as_deref(), "deferred", "stale_score");return None;}if score_progress@@ -455,17 +444,26 @@ impl ApprovalReviewContributor for GuardianV2Extension {> latest_scored_tool_call{thread_store.insert(StrictReviewReason::ElevatedRisk);+ record_fast_decision(extension_metrics.as_deref(), "deferred", "scoring_failure");return None;}- let score = thread_store+ let Some(score) = thread_store.get::<SecurityRiskScore>()- .and_then(|score| score.scores.get("action_risk").copied())?;+ .and_then(|score| score.scores.get("action_risk").copied())+ else {+ record_fast_decision(extension_metrics.as_deref(), "deferred", "missing_score");+ return None;+ };if score < guardian_config.review_threshold {+ record_fast_decision(extension_metrics.as_deref(), "approved", "low_risk");return Some(ReviewDecision::Approved);}if score >= guardian_config.review_threshold {thread_store.insert(StrictReviewReason::ElevatedRisk);+ record_fast_decision(extension_metrics.as_deref(), "deferred", "elevated_risk");+ } else {+ record_fast_decision(extension_metrics.as_deref(), "deferred", "invalid_score");}None})@@ -850,6 +848,7 @@ impl GuardianV2Extension {.latest_scored_tool_call.fetch_max(tool_call_index, Ordering::Release);classification_finished_at = Some(Instant::now());+ record_classification_risk(metrics.as_deref(), output.as_str());if guardian_config.persist_scores&& !config.ephemeral&& let Err(error) = threaddiff --git a/codex-rs/ext/guardian-v2/src/async_scorer/extension_tests.rs b/codex-rs/ext/guardian-v2/src/async_scorer/extension_tests.rsindex c646878506..28a9973ac7 100644--- a/codex-rs/ext/guardian-v2/src/async_scorer/extension_tests.rs+++ b/codex-rs/ext/guardian-v2/src/async_scorer/extension_tests.rs@@ -55,19 +55,21 @@ use core_test_support::test_codex::test_codex;use pretty_assertions::assert_eq;use serde_json::json;-use super::CLASSIFICATION_DURATION_METRIC;-use super::CLASSIFICATION_METRIC;use super::GuardianV2Extension;use super::GuardianV2ScoreProgress;-use super::REVIEW_FALLBACK_METRIC;use super::StrictReviewReason;-use super::TOOL_CALL_LAG_METRIC;use super::encrypted_parent_compaction;use super::should_classify_tool;use crate::async_scorer::config::CLASSIFICATION_OUTPUT_INSTRUCTIONS;use crate::async_scorer::config::DEFAULT_MODEL_CONTEXT_ITEM_TOKENS;use crate::async_scorer::config::DEFAULT_PARENT_COMPACTION_TOKENS;use crate::async_scorer::config::GuardianV2ReviewScope;+use crate::async_scorer::metrics::CLASSIFICATION_DURATION_METRIC;+use crate::async_scorer::metrics::CLASSIFICATION_METRIC;+use crate::async_scorer::metrics::CLASSIFICATION_RISK_METRIC;+use crate::async_scorer::metrics::FAST_DECISION_METRIC;+use crate::async_scorer::metrics::REVIEW_FALLBACK_METRIC;+use crate::async_scorer::metrics::TOOL_CALL_LAG_METRIC;use crate::async_scorer::sampler::CLASSIFICATION_TOKEN_USAGE_METRIC;use crate::async_scorer::sampler::INITIAL_WEBSOCKET_CONNECTIONS;use crate::async_scorer::sampler::LunaSampler;@@ -275,12 +277,23 @@ async fn installed_extension_reconnects_after_auth_refresh() -> Result<()> {Ok(())}-#[derive(Debug, PartialEq)]+#[derive(Clone, Debug, PartialEq)]enum RecordedMetric {Histogram(String, i64, Vec<(String, String)>),Counter(String, i64, Vec<(String, String)>),}+fn fast_decision_metric(decision: &str, reason: &str) -> RecordedMetric {+ RecordedMetric::Counter(+ FAST_DECISION_METRIC.to_owned(),+ 1,+ vec![+ ("decision".to_owned(), decision.to_owned()),+ ("reason".to_owned(), reason.to_owned()),+ ],+ )+}+#[derive(Default)]struct RecordingMetrics(Mutex<Vec<RecordedMetric>>);@@ -487,6 +500,7 @@ async fn computer_use_only_scores_cannot_approve_other_actions() -> Result<()> {action: None,sampled_at: None,});+ thread_store.insert(RecordingMetrics::default());let progress = thread_store.get::<GuardianV2ScoreProgress>().expect("Guardian v2 should track score progress per thread");@@ -542,7 +556,9 @@ async fn computer_use_only_scores_cannot_approve_other_actions() -> Result<()> {&fixture.session_store,thread_store,&prompt,- /*extension_metrics*/ None,+ thread_store+ .get::<RecordingMetrics>()+ .map(|metrics| metrics as Arc<dyn ExtensionMetrics>),).await,expected,@@ -556,12 +572,56 @@ async fn computer_use_only_scores_cannot_approve_other_actions() -> Result<()> {&fixture.session_store,thread_store,"not valid JSON",- /*extension_metrics*/ None,+ thread_store+ .get::<RecordingMetrics>()+ .map(|metrics| metrics as Arc<dyn ExtensionMetrics>),).await,None,"malformed approval actions must not reuse a browser score");+ thread_store.remove::<SecurityRiskScore>();+ assert_eq!(+ fixture+ .registry+ .fast_approval_decision(+ &fixture.session_store,+ thread_store,+ &json!({"tool": "mcp_tool_call", "server": "node_repl"}).to_string(),+ thread_store+ .get::<RecordingMetrics>()+ .map(|metrics| metrics as Arc<dyn ExtensionMetrics>),+ )+ .await,+ None+ );++ let fast_decisions = thread_store+ .get::<RecordingMetrics>()+ .unwrap()+ .0+ .lock()+ .unwrap()+ .iter()+ .filter(|sample| {+ matches!(+ sample,+ RecordedMetric::Counter(name, 1, _) if name == FAST_DECISION_METRIC+ )+ })+ .cloned()+ .collect::<Vec<_>>();+ assert_eq!(+ fast_decisions,+ vec![+ fast_decision_metric("approved", "low_risk"),+ fast_decision_metric("approved", "low_risk"),+ fast_decision_metric("deferred", "out_of_scope"),+ fast_decision_metric("deferred", "out_of_scope"),+ fast_decision_metric("deferred", "out_of_scope"),+ fast_decision_metric("deferred", "missing_score"),+ ]+ );Ok(())}@@ -918,8 +978,16 @@ impl GuardianFailureFixture {).await?;let thread_store = test.codex.thread_extension_data();+ let score_progress = thread_store+ .get::<GuardianV2ScoreProgress>()+ .expect("Guardian v2 should track score progress per thread");tokio::time::timeout(Duration::from_secs(5), async {- while thread_store.get::<SecurityRiskScore>().is_none() {+ while thread_store.get::<SecurityRiskScore>().is_none()+ || score_progress+ .latest_scored_tool_call+ .load(Ordering::Acquire)+ < score_progress.latest_tool_call.load(Ordering::Acquire)+ {tokio::task::yield_now().await;}})@@ -960,7 +1028,7 @@ impl GuardianFailureFixture {.await;}- async fn assert_fails_closed(&self) -> Result<()> {+ async fn assert_fails_closed(&self, expected_reason: &str) -> Result<()> {let thread_store = self.test.codex.thread_extension_data();let score_progress = thread_store.get::<GuardianV2ScoreProgress>()@@ -980,17 +1048,30 @@ impl GuardianFailureFixture {}}).await?;+ thread_store.insert(RecordingMetrics::default());assert_eq!(self.registry.fast_approval_decision(&self.session_store,thread_store,"review action",- /*extension_metrics*/ None,+ thread_store+ .get::<RecordingMetrics>()+ .map(|metrics| metrics as Arc<dyn ExtensionMetrics>),).await,None);+ assert!(+ thread_store+ .get::<RecordingMetrics>()+ .unwrap()+ .0+ .lock()+ .unwrap()+ .iter()+ .any(|sample| sample == &fast_decision_metric("deferred", expected_reason))+ );Ok(())}}@@ -1008,7 +1089,7 @@ async fn contributor_fails_closed_when_thread_lookup_fails() -> Result<()> {.expect("the test thread should exist before simulating a failed lookup");fixture.score_tool(ToolName::plain("read_file")).await;- fixture.assert_fails_closed().await+ fixture.assert_fails_closed("scoring_failure").await}#[tokio::test(flavor = "multi_thread", worker_threads = 2)]@@ -1037,7 +1118,7 @@ async fn contributor_fails_closed_when_model_configuration_is_invalid() -> Resul.insert(parent_model);fixture.score_tool(ToolName::plain("read_file")).await;- fixture.assert_fails_closed().await+ fixture.assert_fails_closed("elevated_risk").await}#[tokio::test(flavor = "multi_thread", worker_threads = 2)]@@ -1050,7 +1131,7 @@ async fn contributor_fails_closed_when_action_serialization_fails() -> Result<());fixture.score_tool(oversized_tool_name).await;- fixture.assert_fails_closed().await+ fixture.assert_fails_closed("elevated_risk").await}#[tokio::test(flavor = "multi_thread", worker_threads = 2)]@@ -1090,7 +1171,7 @@ async fn contributor_fails_closed_when_luna_classification_fails() -> Result<()>);fixture.score_tool(ToolName::plain("read_file")).await;- fixture.assert_fails_closed().await+ fixture.assert_fails_closed("elevated_risk").await}#[tokio::test(flavor = "multi_thread", worker_threads = 2)]@@ -1294,7 +1375,7 @@ max_recent_non_user_entries = 8.latest_scored_tool_call.load(Ordering::Acquire)== 0- || metrics.0.lock().unwrap().len() < 9+ || metrics.0.lock().unwrap().len() < 10{tokio::task::yield_now().await;}@@ -1408,7 +1489,7 @@ max_recent_non_user_entries = 8);let samples = initial_metrics.0.lock().unwrap();- let classification_duration_ms = match &samples[8] {+ let classification_duration_ms = match &samples[9] {RecordedMetric::Histogram(name, duration_ms, _)if name == CLASSIFICATION_DURATION_METRIC =>{@@ -1436,6 +1517,11 @@ max_recent_non_user_entries = 8)}).chain([+ RecordedMetric::Counter(+ CLASSIFICATION_RISK_METRIC.to_owned(),+ 1,+ vec![("risk_level".to_owned(), "high".to_owned())],+ ),RecordedMetric::Counter(CLASSIFICATION_METRIC.to_owned(),1,@@ -1476,8 +1562,11 @@ max_recent_non_user_entries = 8).chain([RecordedMetric::Histogram(TOOL_CALL_LAG_METRIC.to_owned(), 0, vec![]),+ fast_decision_metric("deferred", "elevated_risk"),RecordedMetric::Histogram(TOOL_CALL_LAG_METRIC.to_owned(), 0, vec![]),+ fast_decision_metric("approved", "low_risk"),RecordedMetric::Histogram(TOOL_CALL_LAG_METRIC.to_owned(), 2, vec![]),+ fast_decision_metric("approved", "low_risk"),]).collect::<Vec<_>>());@@ -1491,7 +1580,9 @@ max_recent_non_user_entries = 81,vec![("fallback_reason".to_owned(), "score_lag".to_owned())],),+ fast_decision_metric("deferred", "stale_score"),RecordedMetric::Histogram(TOOL_CALL_LAG_METRIC.to_owned(), 2, vec![]),+ fast_decision_metric("approved", "low_risk"),]);diff --git a/codex-rs/ext/guardian-v2/src/async_scorer/metrics.rs b/codex-rs/ext/guardian-v2/src/async_scorer/metrics.rsnew file mode 100644index 0000000000..235e6f6e0e--- /dev/null+++ b/codex-rs/ext/guardian-v2/src/async_scorer/metrics.rs@@ -0,0 +1,56 @@+//! Low-cardinality metrics for Guardian V2 classification and approval decisions.++use std::time::Duration;++use codex_extension_api::ExtensionMetrics;++pub(super) const CLASSIFICATION_METRIC: &str = "codex.guardian_v2.classification";+pub(super) const CLASSIFICATION_DURATION_METRIC: &str =+ "codex.guardian_v2.classification.duration_ms";+pub(super) const CLASSIFICATION_RISK_METRIC: &str = "codex.guardian_v2.classification.risk";+pub(super) const FAST_DECISION_METRIC: &str = "codex.guardian_v2.fast_decision";+pub(super) const REVIEW_FALLBACK_METRIC: &str = "codex.guardian_v2.review_fallback";+pub(super) const TOOL_CALL_LAG_METRIC: &str = "codex.guardian_v2.tool_call_lag";++pub(super) fn record_classification(+ metrics: Option<&dyn ExtensionMetrics>,+ duration: Duration,+ outcome: &str,+) {+ let Some(metrics) = metrics else {+ return;+ };+ let tags = [("outcome", outcome)];+ metrics.counter(CLASSIFICATION_METRIC, /*inc*/ 1, &tags);+ metrics.histogram(+ CLASSIFICATION_DURATION_METRIC,+ i64::try_from(duration.as_millis()).unwrap_or(i64::MAX),+ &tags,+ );+}++pub(super) fn record_classification_risk(metrics: Option<&dyn ExtensionMetrics>, risk_level: &str) {+ let Some(metrics) = metrics else {+ return;+ };+ metrics.counter(+ CLASSIFICATION_RISK_METRIC,+ /*inc*/ 1,+ &[("risk_level", risk_level)],+ );+}++pub(super) fn record_fast_decision(+ metrics: Option<&dyn ExtensionMetrics>,+ decision: &str,+ reason: &str,+) {+ let Some(metrics) = metrics else {+ return;+ };+ metrics.counter(+ FAST_DECISION_METRIC,+ /*inc*/ 1,+ &[("decision", decision), ("reason", reason)],+ );+}diff --git a/codex-rs/ext/guardian-v2/src/async_scorer/mod.rs b/codex-rs/ext/guardian-v2/src/async_scorer/mod.rsindex b22eae07f4..b552b68abb 100644--- a/codex-rs/ext/guardian-v2/src/async_scorer/mod.rs+++ b/codex-rs/ext/guardian-v2/src/async_scorer/mod.rs@@ -1,5 +1,6 @@mod config;mod extension;+mod metrics;mod review_evidence;mod sampler;mod transcript;