Skip to content

armgit / codex

Public Imported from GitHub
0

Lightweight coding agent that runs in your terminal

Rust

Add Guardian V2 decision metrics (#41100)

jife9a446d79dc2
diff --git a/codex-rs/ext/guardian-v2/src/async_scorer/extension.rs b/codex-rs/ext/guardian-v2/src/async_scorer/extension.rs
index d1e7202770..e3b1277615 100644
--- a/codex-rs/ext/guardian-v2/src/async_scorer/extension.rs
+++ b/codex-rs/ext/guardian-v2/src/async_scorer/extension.rs
@@ -3,7 +3,6 @@ use std::sync::Arc;
use std::sync::Weak;
use std::sync::atomic::AtomicUsize;
use std::sync::atomic::Ordering;
-use std::time::Duration;
use std::time::Instant;
use std::time::SystemTime;
@@ -46,6 +45,11 @@ use serde_json::json;
use super::config::GuardianV2Config;
use super::config::GuardianV2ReviewScope;
+use super::metrics::REVIEW_FALLBACK_METRIC;
+use super::metrics::TOOL_CALL_LAG_METRIC;
+use super::metrics::record_classification;
+use super::metrics::record_classification_risk;
+use super::metrics::record_fast_decision;
use super::review_evidence::render_review_evidence;
use super::sampler::LunaSampler;
use super::sampler::LunaSamplerConfig;
@@ -254,12 +258,6 @@ pub enum StrictReviewReason {
struct GuardianV2Enabled;
-// Sampled when the approval contributor evaluates the cached Luna score.
-const TOOL_CALL_LAG_METRIC: &str = "codex.guardian_v2.tool_call_lag";
-const REVIEW_FALLBACK_METRIC: &str = "codex.guardian_v2.review_fallback";
-const CLASSIFICATION_METRIC: &str = "codex.guardian_v2.classification";
-const CLASSIFICATION_DURATION_METRIC: &str = "codex.guardian_v2.classification.duration_ms";
-
#[derive(Default)]
struct GuardianV2ScoreProgress {
latest_tool_call: AtomicUsize,
@@ -268,23 +266,6 @@ struct GuardianV2ScoreProgress {
metrics: Option<Arc<dyn ExtensionMetrics>>,
}
-fn record_classification(
- metrics: Option<&dyn ExtensionMetrics>,
- duration: Duration,
- outcome: &str,
-) {
- let Some(metrics) = metrics else {
- return;
- };
- let tags = [("outcome", outcome)];
- metrics.counter(CLASSIFICATION_METRIC, /*inc*/ 1, &tags);
- metrics.histogram(
- CLASSIFICATION_DURATION_METRIC,
- i64::try_from(duration.as_millis()).unwrap_or(i64::MAX),
- &tags,
- );
-}
-
#[derive(Clone)]
struct GuardianV2Extension {
auth_manager: Arc<AuthManager>,
@@ -413,17 +394,24 @@ impl ApprovalReviewContributor for GuardianV2Extension {
thread_store.get::<GuardianV2Enabled>()?;
let guardian_config = thread_store.get::<GuardianV2Config>()?;
if guardian_config.review_scope == GuardianV2ReviewScope::ComputerUseOnly {
- let action = serde_json::from_str::<serde_json::Value>(prompt).ok()?;
+ let Ok(action) = serde_json::from_str::<serde_json::Value>(prompt) else {
+ record_fast_decision(extension_metrics.as_deref(), "deferred", "out_of_scope");
+ return None;
+ };
if action.get("tool").and_then(serde_json::Value::as_str) != Some("mcp_tool_call")
|| !action
.get("server")
.and_then(serde_json::Value::as_str)
.is_some_and(is_node_repl_backed_server)
{
+ record_fast_decision(extension_metrics.as_deref(), "deferred", "out_of_scope");
return None;
}
}
- let score_progress = thread_store.get::<GuardianV2ScoreProgress>()?;
+ let Some(score_progress) = thread_store.get::<GuardianV2ScoreProgress>() else {
+ record_fast_decision(extension_metrics.as_deref(), "deferred", "missing_score");
+ return None;
+ };
let latest_scored_tool_call = score_progress
.latest_scored_tool_call
.load(Ordering::Acquire);
@@ -447,6 +435,7 @@ impl ApprovalReviewContributor for GuardianV2Extension {
&[("fallback_reason", "score_lag")],
);
}
+ record_fast_decision(extension_metrics.as_deref(), "deferred", "stale_score");
return None;
}
if score_progress
@@ -455,17 +444,26 @@ impl ApprovalReviewContributor for GuardianV2Extension {
> latest_scored_tool_call
{
thread_store.insert(StrictReviewReason::ElevatedRisk);
+ record_fast_decision(extension_metrics.as_deref(), "deferred", "scoring_failure");
return None;
}
- let score = thread_store
+ let Some(score) = thread_store
.get::<SecurityRiskScore>()
- .and_then(|score| score.scores.get("action_risk").copied())?;
+ .and_then(|score| score.scores.get("action_risk").copied())
+ else {
+ record_fast_decision(extension_metrics.as_deref(), "deferred", "missing_score");
+ return None;
+ };
if score < guardian_config.review_threshold {
+ record_fast_decision(extension_metrics.as_deref(), "approved", "low_risk");
return Some(ReviewDecision::Approved);
}
if score >= guardian_config.review_threshold {
thread_store.insert(StrictReviewReason::ElevatedRisk);
+ record_fast_decision(extension_metrics.as_deref(), "deferred", "elevated_risk");
+ } else {
+ record_fast_decision(extension_metrics.as_deref(), "deferred", "invalid_score");
}
None
})
@@ -850,6 +848,7 @@ impl GuardianV2Extension {
.latest_scored_tool_call
.fetch_max(tool_call_index, Ordering::Release);
classification_finished_at = Some(Instant::now());
+ record_classification_risk(metrics.as_deref(), output.as_str());
if guardian_config.persist_scores
&& !config.ephemeral
&& let Err(error) = thread
diff --git a/codex-rs/ext/guardian-v2/src/async_scorer/extension_tests.rs b/codex-rs/ext/guardian-v2/src/async_scorer/extension_tests.rs
index c646878506..28a9973ac7 100644
--- a/codex-rs/ext/guardian-v2/src/async_scorer/extension_tests.rs
+++ b/codex-rs/ext/guardian-v2/src/async_scorer/extension_tests.rs
@@ -55,19 +55,21 @@ use core_test_support::test_codex::test_codex;
use pretty_assertions::assert_eq;
use serde_json::json;
-use super::CLASSIFICATION_DURATION_METRIC;
-use super::CLASSIFICATION_METRIC;
use super::GuardianV2Extension;
use super::GuardianV2ScoreProgress;
-use super::REVIEW_FALLBACK_METRIC;
use super::StrictReviewReason;
-use super::TOOL_CALL_LAG_METRIC;
use super::encrypted_parent_compaction;
use super::should_classify_tool;
use crate::async_scorer::config::CLASSIFICATION_OUTPUT_INSTRUCTIONS;
use crate::async_scorer::config::DEFAULT_MODEL_CONTEXT_ITEM_TOKENS;
use crate::async_scorer::config::DEFAULT_PARENT_COMPACTION_TOKENS;
use crate::async_scorer::config::GuardianV2ReviewScope;
+use crate::async_scorer::metrics::CLASSIFICATION_DURATION_METRIC;
+use crate::async_scorer::metrics::CLASSIFICATION_METRIC;
+use crate::async_scorer::metrics::CLASSIFICATION_RISK_METRIC;
+use crate::async_scorer::metrics::FAST_DECISION_METRIC;
+use crate::async_scorer::metrics::REVIEW_FALLBACK_METRIC;
+use crate::async_scorer::metrics::TOOL_CALL_LAG_METRIC;
use crate::async_scorer::sampler::CLASSIFICATION_TOKEN_USAGE_METRIC;
use crate::async_scorer::sampler::INITIAL_WEBSOCKET_CONNECTIONS;
use crate::async_scorer::sampler::LunaSampler;
@@ -275,12 +277,23 @@ async fn installed_extension_reconnects_after_auth_refresh() -> Result<()> {
Ok(())
}
-#[derive(Debug, PartialEq)]
+#[derive(Clone, Debug, PartialEq)]
enum RecordedMetric {
Histogram(String, i64, Vec<(String, String)>),
Counter(String, i64, Vec<(String, String)>),
}
+fn fast_decision_metric(decision: &str, reason: &str) -> RecordedMetric {
+ RecordedMetric::Counter(
+ FAST_DECISION_METRIC.to_owned(),
+ 1,
+ vec![
+ ("decision".to_owned(), decision.to_owned()),
+ ("reason".to_owned(), reason.to_owned()),
+ ],
+ )
+}
+
#[derive(Default)]
struct RecordingMetrics(Mutex<Vec<RecordedMetric>>);
@@ -487,6 +500,7 @@ async fn computer_use_only_scores_cannot_approve_other_actions() -> Result<()> {
action: None,
sampled_at: None,
});
+ thread_store.insert(RecordingMetrics::default());
let progress = thread_store
.get::<GuardianV2ScoreProgress>()
.expect("Guardian v2 should track score progress per thread");
@@ -542,7 +556,9 @@ async fn computer_use_only_scores_cannot_approve_other_actions() -> Result<()> {
&fixture.session_store,
thread_store,
&prompt,
- /*extension_metrics*/ None,
+ thread_store
+ .get::<RecordingMetrics>()
+ .map(|metrics| metrics as Arc<dyn ExtensionMetrics>),
)
.await,
expected,
@@ -556,12 +572,56 @@ async fn computer_use_only_scores_cannot_approve_other_actions() -> Result<()> {
&fixture.session_store,
thread_store,
"not valid JSON",
- /*extension_metrics*/ None,
+ thread_store
+ .get::<RecordingMetrics>()
+ .map(|metrics| metrics as Arc<dyn ExtensionMetrics>),
)
.await,
None,
"malformed approval actions must not reuse a browser score"
);
+ thread_store.remove::<SecurityRiskScore>();
+ assert_eq!(
+ fixture
+ .registry
+ .fast_approval_decision(
+ &fixture.session_store,
+ thread_store,
+ &json!({"tool": "mcp_tool_call", "server": "node_repl"}).to_string(),
+ thread_store
+ .get::<RecordingMetrics>()
+ .map(|metrics| metrics as Arc<dyn ExtensionMetrics>),
+ )
+ .await,
+ None
+ );
+
+ let fast_decisions = thread_store
+ .get::<RecordingMetrics>()
+ .unwrap()
+ .0
+ .lock()
+ .unwrap()
+ .iter()
+ .filter(|sample| {
+ matches!(
+ sample,
+ RecordedMetric::Counter(name, 1, _) if name == FAST_DECISION_METRIC
+ )
+ })
+ .cloned()
+ .collect::<Vec<_>>();
+ assert_eq!(
+ fast_decisions,
+ vec![
+ fast_decision_metric("approved", "low_risk"),
+ fast_decision_metric("approved", "low_risk"),
+ fast_decision_metric("deferred", "out_of_scope"),
+ fast_decision_metric("deferred", "out_of_scope"),
+ fast_decision_metric("deferred", "out_of_scope"),
+ fast_decision_metric("deferred", "missing_score"),
+ ]
+ );
Ok(())
}
@@ -918,8 +978,16 @@ impl GuardianFailureFixture {
)
.await?;
let thread_store = test.codex.thread_extension_data();
+ let score_progress = thread_store
+ .get::<GuardianV2ScoreProgress>()
+ .expect("Guardian v2 should track score progress per thread");
tokio::time::timeout(Duration::from_secs(5), async {
- while thread_store.get::<SecurityRiskScore>().is_none() {
+ while thread_store.get::<SecurityRiskScore>().is_none()
+ || score_progress
+ .latest_scored_tool_call
+ .load(Ordering::Acquire)
+ < score_progress.latest_tool_call.load(Ordering::Acquire)
+ {
tokio::task::yield_now().await;
}
})
@@ -960,7 +1028,7 @@ impl GuardianFailureFixture {
.await;
}
- async fn assert_fails_closed(&self) -> Result<()> {
+ async fn assert_fails_closed(&self, expected_reason: &str) -> Result<()> {
let thread_store = self.test.codex.thread_extension_data();
let score_progress = thread_store
.get::<GuardianV2ScoreProgress>()
@@ -980,17 +1048,30 @@ impl GuardianFailureFixture {
}
})
.await?;
+ thread_store.insert(RecordingMetrics::default());
assert_eq!(
self.registry
.fast_approval_decision(
&self.session_store,
thread_store,
"review action",
- /*extension_metrics*/ None,
+ thread_store
+ .get::<RecordingMetrics>()
+ .map(|metrics| metrics as Arc<dyn ExtensionMetrics>),
)
.await,
None
);
+ assert!(
+ thread_store
+ .get::<RecordingMetrics>()
+ .unwrap()
+ .0
+ .lock()
+ .unwrap()
+ .iter()
+ .any(|sample| sample == &fast_decision_metric("deferred", expected_reason))
+ );
Ok(())
}
}
@@ -1008,7 +1089,7 @@ async fn contributor_fails_closed_when_thread_lookup_fails() -> Result<()> {
.expect("the test thread should exist before simulating a failed lookup");
fixture.score_tool(ToolName::plain("read_file")).await;
- fixture.assert_fails_closed().await
+ fixture.assert_fails_closed("scoring_failure").await
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
@@ -1037,7 +1118,7 @@ async fn contributor_fails_closed_when_model_configuration_is_invalid() -> Resul
.insert(parent_model);
fixture.score_tool(ToolName::plain("read_file")).await;
- fixture.assert_fails_closed().await
+ fixture.assert_fails_closed("elevated_risk").await
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
@@ -1050,7 +1131,7 @@ async fn contributor_fails_closed_when_action_serialization_fails() -> Result<()
);
fixture.score_tool(oversized_tool_name).await;
- fixture.assert_fails_closed().await
+ fixture.assert_fails_closed("elevated_risk").await
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
@@ -1090,7 +1171,7 @@ async fn contributor_fails_closed_when_luna_classification_fails() -> Result<()>
);
fixture.score_tool(ToolName::plain("read_file")).await;
- fixture.assert_fails_closed().await
+ fixture.assert_fails_closed("elevated_risk").await
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
@@ -1294,7 +1375,7 @@ max_recent_non_user_entries = 8
.latest_scored_tool_call
.load(Ordering::Acquire)
== 0
- || metrics.0.lock().unwrap().len() < 9
+ || metrics.0.lock().unwrap().len() < 10
{
tokio::task::yield_now().await;
}
@@ -1408,7 +1489,7 @@ max_recent_non_user_entries = 8
);
let samples = initial_metrics.0.lock().unwrap();
- let classification_duration_ms = match &samples[8] {
+ let classification_duration_ms = match &samples[9] {
RecordedMetric::Histogram(name, duration_ms, _)
if name == CLASSIFICATION_DURATION_METRIC =>
{
@@ -1436,6 +1517,11 @@ max_recent_non_user_entries = 8
)
})
.chain([
+ RecordedMetric::Counter(
+ CLASSIFICATION_RISK_METRIC.to_owned(),
+ 1,
+ vec![("risk_level".to_owned(), "high".to_owned())],
+ ),
RecordedMetric::Counter(
CLASSIFICATION_METRIC.to_owned(),
1,
@@ -1476,8 +1562,11 @@ max_recent_non_user_entries = 8
)
.chain([
RecordedMetric::Histogram(TOOL_CALL_LAG_METRIC.to_owned(), 0, vec![]),
+ fast_decision_metric("deferred", "elevated_risk"),
RecordedMetric::Histogram(TOOL_CALL_LAG_METRIC.to_owned(), 0, vec![]),
+ fast_decision_metric("approved", "low_risk"),
RecordedMetric::Histogram(TOOL_CALL_LAG_METRIC.to_owned(), 2, vec![]),
+ fast_decision_metric("approved", "low_risk"),
])
.collect::<Vec<_>>()
);
@@ -1491,7 +1580,9 @@ max_recent_non_user_entries = 8
1,
vec![("fallback_reason".to_owned(), "score_lag".to_owned())],
),
+ fast_decision_metric("deferred", "stale_score"),
RecordedMetric::Histogram(TOOL_CALL_LAG_METRIC.to_owned(), 2, vec![]),
+ fast_decision_metric("approved", "low_risk"),
]
);
diff --git a/codex-rs/ext/guardian-v2/src/async_scorer/metrics.rs b/codex-rs/ext/guardian-v2/src/async_scorer/metrics.rs
new file mode 100644
index 0000000000..235e6f6e0e
--- /dev/null
+++ b/codex-rs/ext/guardian-v2/src/async_scorer/metrics.rs
@@ -0,0 +1,56 @@
+//! Low-cardinality metrics for Guardian V2 classification and approval decisions.
+
+use std::time::Duration;
+
+use codex_extension_api::ExtensionMetrics;
+
+pub(super) const CLASSIFICATION_METRIC: &str = "codex.guardian_v2.classification";
+pub(super) const CLASSIFICATION_DURATION_METRIC: &str =
+ "codex.guardian_v2.classification.duration_ms";
+pub(super) const CLASSIFICATION_RISK_METRIC: &str = "codex.guardian_v2.classification.risk";
+pub(super) const FAST_DECISION_METRIC: &str = "codex.guardian_v2.fast_decision";
+pub(super) const REVIEW_FALLBACK_METRIC: &str = "codex.guardian_v2.review_fallback";
+pub(super) const TOOL_CALL_LAG_METRIC: &str = "codex.guardian_v2.tool_call_lag";
+
+pub(super) fn record_classification(
+ metrics: Option<&dyn ExtensionMetrics>,
+ duration: Duration,
+ outcome: &str,
+) {
+ let Some(metrics) = metrics else {
+ return;
+ };
+ let tags = [("outcome", outcome)];
+ metrics.counter(CLASSIFICATION_METRIC, /*inc*/ 1, &tags);
+ metrics.histogram(
+ CLASSIFICATION_DURATION_METRIC,
+ i64::try_from(duration.as_millis()).unwrap_or(i64::MAX),
+ &tags,
+ );
+}
+
+pub(super) fn record_classification_risk(metrics: Option<&dyn ExtensionMetrics>, risk_level: &str) {
+ let Some(metrics) = metrics else {
+ return;
+ };
+ metrics.counter(
+ CLASSIFICATION_RISK_METRIC,
+ /*inc*/ 1,
+ &[("risk_level", risk_level)],
+ );
+}
+
+pub(super) fn record_fast_decision(
+ metrics: Option<&dyn ExtensionMetrics>,
+ decision: &str,
+ reason: &str,
+) {
+ let Some(metrics) = metrics else {
+ return;
+ };
+ metrics.counter(
+ FAST_DECISION_METRIC,
+ /*inc*/ 1,
+ &[("decision", decision), ("reason", reason)],
+ );
+}
diff --git a/codex-rs/ext/guardian-v2/src/async_scorer/mod.rs b/codex-rs/ext/guardian-v2/src/async_scorer/mod.rs
index b22eae07f4..b552b68abb 100644
--- a/codex-rs/ext/guardian-v2/src/async_scorer/mod.rs
+++ b/codex-rs/ext/guardian-v2/src/async_scorer/mod.rs
@@ -1,5 +1,6 @@
mod config;
mod extension;
+mod metrics;
mod review_evidence;
mod sampler;
mod transcript;