Skip to content

armgit / codex

Public Imported from GitHub
0

Lightweight coding agent that runs in your terminal

Rust

Add Guardian v2 approval review metrics (#39224)

Dylan Hurde7e13c68e224
diff --git a/codex-rs/core/src/guardian/review.rs b/codex-rs/core/src/guardian/review.rs
index a3c37b650a..68f3f9d058 100644
--- a/codex-rs/core/src/guardian/review.rs
+++ b/codex-rs/core/src/guardian/review.rs
@@ -330,6 +330,9 @@ async fn run_guardian_review(
&session.services.session_extension_data,
&session.services.thread_extension_data,
&action.to_string(),
+ Some(crate::session::extension_metrics::from_session_telemetry(
+ turn.session_telemetry.clone(),
+ )),
)
.await
{
diff --git a/codex-rs/core/src/guardian/tests.rs b/codex-rs/core/src/guardian/tests.rs
index 4da79a5eb5..a851fefcee 100644
--- a/codex-rs/core/src/guardian/tests.rs
+++ b/codex-rs/core/src/guardian/tests.rs
@@ -2999,6 +2999,7 @@ async fn escalated_retry_bypasses_extension_approval_and_runs_guardian() -> anyh
_session_store: &'a codex_extension_api::ExtensionData,
_thread_store: &'a codex_extension_api::ExtensionData,
_prompt: &'a str,
+ _extension_metrics: Option<Arc<dyn codex_extension_api::ExtensionMetrics>>,
) -> codex_extension_api::ExtensionFuture<'a, Option<ReviewDecision>> {
Box::pin(async move { Some(ReviewDecision::Approved) })
}
diff --git a/codex-rs/core/src/session/extension_metrics.rs b/codex-rs/core/src/session/extension_metrics.rs
index d4a30b7717..13547c6bc2 100644
--- a/codex-rs/core/src/session/extension_metrics.rs
+++ b/codex-rs/core/src/session/extension_metrics.rs
@@ -8,12 +8,16 @@ struct SessionTelemetryExtensionMetrics {
}
impl ExtensionMetrics for SessionTelemetryExtensionMetrics {
+ fn counter(&self, name: &str, inc: i64, tags: &[(&str, &str)]) {
+ self.session_telemetry.counter(name, inc, tags);
+ }
+
fn histogram(&self, name: &str, value: i64, tags: &[(&str, &str)]) {
self.session_telemetry.histogram(name, value, tags);
}
}
-pub(super) fn from_session_telemetry(
+pub(crate) fn from_session_telemetry(
session_telemetry: SessionTelemetry,
) -> Arc<dyn ExtensionMetrics> {
Arc::new(SessionTelemetryExtensionMetrics { session_telemetry })
diff --git a/codex-rs/core/src/session/mod.rs b/codex-rs/core/src/session/mod.rs
index 1fe8942b84..3cd72aabb7 100644
--- a/codex-rs/core/src/session/mod.rs
+++ b/codex-rs/core/src/session/mod.rs
@@ -204,7 +204,7 @@ use codex_protocol::exec_output::StreamOutput;
mod code_mode_warning;
pub(crate) mod context_window;
mod environment;
-mod extension_metrics;
+pub(crate) mod extension_metrics;
mod handlers;
mod inject;
mod input_queue;
diff --git a/codex-rs/core/src/session/tests.rs b/codex-rs/core/src/session/tests.rs
index dfde550e51..222ea7d639 100644
--- a/codex-rs/core/src/session/tests.rs
+++ b/codex-rs/core/src/session/tests.rs
@@ -402,8 +402,31 @@ fn extension_metrics_preserve_session_metadata_tags() {
],
);
+ extension_metrics.counter(
+ "codex.test.extension.counter",
+ /*inc*/ 2,
+ &[("component", "skills"), ("model", "extension-model")],
+ );
+
let snapshot = metrics.snapshot().expect("metrics snapshot");
let attributes = single_histogram_attributes(&snapshot, "codex.test.extension");
+ let counter = find_metric(&snapshot, "codex.test.extension.counter");
+ let AggregatedMetrics::U64(MetricData::Sum(sum)) = counter.data() else {
+ panic!("expected counter");
+ };
+ let points = sum.data_points().collect::<Vec<_>>();
+ assert_eq!(points.len(), 1);
+ assert_eq!(points[0].value(), 2);
+ assert_eq!(
+ points[0]
+ .attributes()
+ .map(|attribute| (
+ attribute.key.as_str().to_string(),
+ attribute.value.as_str().to_string(),
+ ))
+ .collect::<BTreeMap<_, _>>(),
+ attributes,
+ );
assert_eq!(
attributes,
BTreeMap::from([
diff --git a/codex-rs/core/tests/suite/auto_review.rs b/codex-rs/core/tests/suite/auto_review.rs
index d83561155e..f0dd502057 100644
--- a/codex-rs/core/tests/suite/auto_review.rs
+++ b/codex-rs/core/tests/suite/auto_review.rs
@@ -6,6 +6,7 @@ use anyhow::Result;
use codex_extension_api::ApprovalReviewContributor;
use codex_extension_api::ExtensionData;
use codex_extension_api::ExtensionFuture;
+use codex_extension_api::ExtensionMetrics;
use codex_extension_api::ExtensionRegistryBuilder;
use codex_features::Feature;
use codex_login::CodexAuth;
@@ -64,8 +65,10 @@ impl ApprovalReviewContributor for ApprovedReviewContributor {
_session_store: &'a ExtensionData,
_thread_store: &'a ExtensionData,
prompt: &'a str,
+ extension_metrics: Option<Arc<dyn ExtensionMetrics>>,
) -> ExtensionFuture<'a, Option<ReviewDecision>> {
Box::pin(async move {
+ assert!(extension_metrics.is_some());
assert!(prompt.contains("\"tool\":\"request_permissions\""));
Some(ReviewDecision::Approved)
})
diff --git a/codex-rs/ext/extension-api/src/capabilities/metrics.rs b/codex-rs/ext/extension-api/src/capabilities/metrics.rs
index 756519ed33..b61b3d9616 100644
--- a/codex-rs/ext/extension-api/src/capabilities/metrics.rs
+++ b/codex-rs/ext/extension-api/src/capabilities/metrics.rs
@@ -3,6 +3,9 @@
/// Implementations are expected to attach the host's session attribution before
/// forwarding samples to the configured metrics backend.
pub trait ExtensionMetrics: Send + Sync {
+ /// Increments a counter with optional extension-provided tags.
+ fn counter(&self, name: &str, inc: i64, tags: &[(&str, &str)]);
+
/// Records one histogram sample with optional extension-provided tags.
fn histogram(&self, name: &str, value: i64, tags: &[(&str, &str)]);
}
diff --git a/codex-rs/ext/extension-api/src/contributors.rs b/codex-rs/ext/extension-api/src/contributors.rs
index ac3fcb1a2f..4c4c3d4522 100644
--- a/codex-rs/ext/extension-api/src/contributors.rs
+++ b/codex-rs/ext/extension-api/src/contributors.rs
@@ -318,11 +318,13 @@ pub trait ToolLifecycleContributor: Send + Sync {
/// Extension contribution that can claim rendered approval-review prompts.
pub trait ApprovalReviewContributor: Send + Sync {
+ /// Reviews one action using metrics bound to the active turn's model.
fn contribute<'a>(
&'a self,
session_store: &'a ExtensionData,
thread_store: &'a ExtensionData,
prompt: &'a str,
+ extension_metrics: Option<Arc<dyn ExtensionMetrics>>,
) -> ExtensionFuture<'a, Option<ReviewDecision>>;
}
diff --git a/codex-rs/ext/extension-api/src/registry.rs b/codex-rs/ext/extension-api/src/registry.rs
index f34c88cc3b..be4670cbba 100644
--- a/codex-rs/ext/extension-api/src/registry.rs
+++ b/codex-rs/ext/extension-api/src/registry.rs
@@ -7,6 +7,7 @@ use crate::ConfigContributor;
use crate::ContextContributor;
use crate::ExtensionData;
use crate::ExtensionEventSink;
+use crate::ExtensionMetrics;
use crate::McpServerContributor;
use crate::NoopExtensionEventSink;
use crate::SkillInvocationContributor;
@@ -194,10 +195,16 @@ impl<C: Sync> ExtensionRegistry<C> {
session_store: &ExtensionData,
thread_store: &ExtensionData,
prompt: &str,
+ extension_metrics: Option<Arc<dyn ExtensionMetrics>>,
) -> Option<ReviewDecision> {
for contributor in &self.approval_review_contributors {
if let Some(decision) = contributor
- .contribute(session_store, thread_store, prompt)
+ .contribute(
+ session_store,
+ thread_store,
+ prompt,
+ extension_metrics.clone(),
+ )
.await
{
return Some(decision);
diff --git a/codex-rs/ext/extension-api/tests/registry.rs b/codex-rs/ext/extension-api/tests/registry.rs
index e5579e5453..129e890589 100644
--- a/codex-rs/ext/extension-api/tests/registry.rs
+++ b/codex-rs/ext/extension-api/tests/registry.rs
@@ -135,6 +135,7 @@ impl ApprovalReviewContributor for AllContributors {
_session_store: &'a ExtensionData,
_thread_store: &'a ExtensionData,
_prompt: &'a str,
+ _extension_metrics: Option<Arc<dyn ExtensionMetrics>>,
) -> ExtensionFuture<'a, Option<ReviewDecision>> {
Box::pin(async move {
let _self = self;
@@ -176,6 +177,7 @@ async fn build_round_trips_every_contributor_category() {
&ExtensionData::new("session"),
&ExtensionData::new("thread"),
"review this",
+ /*extension_metrics*/ None,
)
.await,
Some(ReviewDecision::ApprovedForSession)
@@ -322,6 +324,7 @@ impl ApprovalReviewContributor for RecordingApprovalContributor {
session_store: &'a ExtensionData,
thread_store: &'a ExtensionData,
prompt: &'a str,
+ _extension_metrics: Option<Arc<dyn ExtensionMetrics>>,
) -> ExtensionFuture<'a, Option<ReviewDecision>> {
Box::pin(async move {
self.calls
@@ -363,6 +366,7 @@ async fn approval_review_returns_first_claim_and_short_circuits() {
&ExtensionData::new("session-1"),
&ExtensionData::new("thread-1"),
"allow command?",
+ /*extension_metrics*/ None,
)
.await;
@@ -451,6 +455,7 @@ async fn empty_registry_does_not_claim_approval_review() {
&ExtensionData::new("session"),
&ExtensionData::new("thread"),
"unclaimed",
+ /*extension_metrics*/ None,
)
.await,
None
diff --git a/codex-rs/ext/guardian-v2/src/extension.rs b/codex-rs/ext/guardian-v2/src/extension.rs
index 1a944f010f..fffe9dcef5 100644
--- a/codex-rs/ext/guardian-v2/src/extension.rs
+++ b/codex-rs/ext/guardian-v2/src/extension.rs
@@ -11,6 +11,7 @@ use codex_extension_api::ApprovalReviewContributor;
use codex_extension_api::ExtensionData;
use codex_extension_api::ExtensionEventSink;
use codex_extension_api::ExtensionFuture;
+use codex_extension_api::ExtensionMetrics;
use codex_extension_api::ExtensionRegistryBuilder;
use codex_extension_api::ExtensionWarning;
use codex_extension_api::ResponseItem;
@@ -190,6 +191,10 @@ pub enum StrictReviewReason {
struct GuardianV2Enabled;
+// Sampled when the approval contributor evaluates the cached Luna score.
+const TOOL_CALL_LAG_METRIC: &str = "codex.guardian_v2.tool_call_lag";
+const REVIEW_FALLBACK_METRIC: &str = "codex.guardian_v2.review_fallback";
+
#[derive(Default)]
struct GuardianV2ScoreProgress {
latest_tool_call: AtomicUsize,
@@ -284,6 +289,7 @@ impl ApprovalReviewContributor for GuardianV2Extension {
_session_store: &'a ExtensionData,
thread_store: &'a ExtensionData,
_prompt: &'a str,
+ extension_metrics: Option<Arc<dyn ExtensionMetrics>>,
) -> ExtensionFuture<'a, Option<ReviewDecision>> {
Box::pin(async move {
thread_store.get::<GuardianV2Enabled>()?;
@@ -297,8 +303,22 @@ impl ApprovalReviewContributor for GuardianV2Extension {
.latest_scored_tool_call
.load(Ordering::Acquire),
);
+ if let Some(metrics) = &extension_metrics {
+ metrics.histogram(
+ TOOL_CALL_LAG_METRIC,
+ i64::try_from(tool_call_lag).unwrap_or(i64::MAX),
+ &[],
+ );
+ }
if tool_call_lag > guardian_config.max_tool_call_lag {
thread_store.insert(StrictReviewReason::StaleScore);
+ if let Some(metrics) = &extension_metrics {
+ metrics.counter(
+ REVIEW_FALLBACK_METRIC,
+ /*inc*/ 1,
+ &[("fallback_reason", "score_lag")],
+ );
+ }
return None;
}
diff --git a/codex-rs/ext/guardian-v2/src/extension_tests.rs b/codex-rs/ext/guardian-v2/src/extension_tests.rs
index 57d0821aac..30d1f62ed9 100644
--- a/codex-rs/ext/guardian-v2/src/extension_tests.rs
+++ b/codex-rs/ext/guardian-v2/src/extension_tests.rs
@@ -1,5 +1,6 @@
use std::collections::BTreeMap;
use std::sync::Arc;
+use std::sync::Mutex;
use std::sync::atomic::Ordering;
use std::time::Duration;
@@ -7,6 +8,7 @@ use anyhow::Result;
use codex_core::config::Config;
use codex_extension_api::ConversationHistorySnapshot;
use codex_extension_api::ExtensionData;
+use codex_extension_api::ExtensionMetrics;
use codex_extension_api::ExtensionRegistry;
use codex_extension_api::ExtensionRegistryBuilder;
use codex_extension_api::ResponseItem;
@@ -47,7 +49,9 @@ use pretty_assertions::assert_eq;
use serde_json::json;
use super::GuardianV2ScoreProgress;
+use super::REVIEW_FALLBACK_METRIC;
use super::StrictReviewReason;
+use super::TOOL_CALL_LAG_METRIC;
use super::encrypted_parent_compaction;
use crate::config::DEFAULT_MODEL_CONTEXT_ITEM_TOKENS;
use crate::config::DEFAULT_PARENT_COMPACTION_TOKENS;
@@ -177,6 +181,37 @@ async fn installed_extension_reconnects_after_auth_refresh() -> Result<()> {
Ok(())
}
+#[derive(Debug, PartialEq)]
+enum RecordedMetric {
+ Histogram(String, i64, Vec<(String, String)>),
+ Counter(String, i64, Vec<(String, String)>),
+}
+
+#[derive(Default)]
+struct RecordingMetrics(Mutex<Vec<RecordedMetric>>);
+
+impl ExtensionMetrics for RecordingMetrics {
+ fn counter(&self, name: &str, inc: i64, tags: &[(&str, &str)]) {
+ self.0.lock().unwrap().push(RecordedMetric::Counter(
+ name.to_owned(),
+ inc,
+ tags.iter()
+ .map(|(key, value)| ((*key).to_owned(), (*value).to_owned()))
+ .collect(),
+ ));
+ }
+
+ fn histogram(&self, name: &str, value: i64, tags: &[(&str, &str)]) {
+ self.0.lock().unwrap().push(RecordedMetric::Histogram(
+ name.to_owned(),
+ value,
+ tags.iter()
+ .map(|(key, value)| ((*key).to_owned(), (*value).to_owned()))
+ .collect(),
+ ));
+ }
+}
+
struct TestConversationHistory(Vec<ResponseItem>);
impl ConversationHistorySnapshot for TestConversationHistory {
@@ -425,6 +460,8 @@ async fn sample_configured_conversation_history(
let registry = builder.build();
let session_store = ExtensionData::new("session-1");
let thread_store = test.codex.thread_extension_data();
+ thread_store.insert(RecordingMetrics::default());
+ let metrics = thread_store.get::<RecordingMetrics>().unwrap();
if has_model_defaults {
let parent_model = test
.thread_manager
@@ -435,7 +472,12 @@ async fn sample_configured_conversation_history(
}
assert_eq!(
registry
- .approval_review(&session_store, thread_store, "review action")
+ .approval_review(
+ &session_store,
+ thread_store,
+ "review action",
+ /*extension_metrics*/ None,
+ )
.await,
None
);
@@ -446,7 +488,7 @@ async fn sample_configured_conversation_history(
persistent_thread_state_available: false,
environments: &[],
mcp_resource_client: None,
- extension_metrics: None,
+ extension_metrics: Some(metrics),
session_store: &session_store,
thread_store,
})
@@ -607,7 +649,14 @@ max_recent_non_user_entries = 8
});
assert_eq!(
registry
- .approval_review(&session_store, thread_store, "review action")
+ .approval_review(
+ &session_store,
+ thread_store,
+ "review action",
+ thread_store
+ .get::<RecordingMetrics>()
+ .map(|metrics| metrics as Arc<dyn ExtensionMetrics>),
+ )
.await,
None
);
@@ -621,7 +670,14 @@ max_recent_non_user_entries = 8
});
assert_eq!(
registry
- .approval_review(&session_store, thread_store, "review action")
+ .approval_review(
+ &session_store,
+ thread_store,
+ "review action",
+ thread_store
+ .get::<RecordingMetrics>()
+ .map(|metrics| metrics as Arc<dyn ExtensionMetrics>),
+ )
.await,
Some(ReviewDecision::Approved)
);
@@ -637,17 +693,33 @@ max_recent_non_user_entries = 8
.store(/*val*/ 3, Ordering::Release);
assert_eq!(
registry
- .approval_review(&session_store, thread_store, "review action")
+ .approval_review(
+ &session_store,
+ thread_store,
+ "review action",
+ thread_store
+ .get::<RecordingMetrics>()
+ .map(|metrics| metrics as Arc<dyn ExtensionMetrics>),
+ )
.await,
Some(ReviewDecision::Approved)
);
+ let initial_metrics = thread_store.get::<RecordingMetrics>().unwrap();
+ thread_store.insert(RecordingMetrics::default());
score_progress
.latest_tool_call
.store(/*val*/ 4, Ordering::Release);
assert_eq!(
registry
- .approval_review(&session_store, thread_store, "review action")
+ .approval_review(
+ &session_store,
+ thread_store,
+ "review action",
+ thread_store
+ .get::<RecordingMetrics>()
+ .map(|metrics| metrics as Arc<dyn ExtensionMetrics>),
+ )
.await,
None
);
@@ -661,11 +733,40 @@ max_recent_non_user_entries = 8
.store(/*val*/ 2, Ordering::Release);
assert_eq!(
registry
- .approval_review(&session_store, thread_store, "review action")
+ .approval_review(
+ &session_store,
+ thread_store,
+ "review action",
+ thread_store
+ .get::<RecordingMetrics>()
+ .map(|metrics| metrics as Arc<dyn ExtensionMetrics>),
+ )
.await,
Some(ReviewDecision::Approved)
);
+ assert_eq!(
+ *initial_metrics.0.lock().unwrap(),
+ vec![
+ RecordedMetric::Histogram(TOOL_CALL_LAG_METRIC.to_owned(), 0, vec![]),
+ RecordedMetric::Histogram(TOOL_CALL_LAG_METRIC.to_owned(), 0, vec![]),
+ RecordedMetric::Histogram(TOOL_CALL_LAG_METRIC.to_owned(), 2, vec![]),
+ ]
+ );
+ let metrics = thread_store.get::<RecordingMetrics>().unwrap();
+ assert_eq!(
+ *metrics.0.lock().unwrap(),
+ vec![
+ RecordedMetric::Histogram(TOOL_CALL_LAG_METRIC.to_owned(), 3, vec![]),
+ RecordedMetric::Counter(
+ REVIEW_FALLBACK_METRIC.to_owned(),
+ 1,
+ vec![("fallback_reason".to_owned(), "score_lag".to_owned())],
+ ),
+ RecordedMetric::Histogram(TOOL_CALL_LAG_METRIC.to_owned(), 2, vec![]),
+ ]
+ );
+
Ok(())
}
@@ -847,7 +948,12 @@ async fn contributor_uses_model_defaults_and_preserves_local_overrides() -> Resu
});
assert_eq!(
registry
- .approval_review(&session_store, thread_store, "review action")
+ .approval_review(
+ &session_store,
+ thread_store,
+ "review action",
+ /*extension_metrics*/ None,
+ )
.await,
Some(ReviewDecision::Approved)
);
@@ -980,7 +1086,12 @@ async fn contributor_samples_tool_calls_with_the_existing_luna_pool() -> Result<
assert!(score.sampled_at.is_some());
assert_eq!(
registry
- .approval_review(&session_store, thread_store, "review action")
+ .approval_review(
+ &session_store,
+ thread_store,
+ "review action",
+ /*extension_metrics*/ None,
+ )
.await,
None
);
@@ -1005,7 +1116,12 @@ async fn contributor_samples_tool_calls_with_the_existing_luna_pool() -> Result<
});
assert_eq!(
registry
- .approval_review(&session_store, thread_store, "review action")
+ .approval_review(
+ &session_store,
+ thread_store,
+ "review action",
+ /*extension_metrics*/ None,
+ )
.await,
Some(ReviewDecision::Approved)
);
@@ -1017,7 +1133,12 @@ async fn contributor_samples_tool_calls_with_the_existing_luna_pool() -> Result<
});
assert_eq!(
registry
- .approval_review(&session_store, &disabled_thread_store, "review action")
+ .approval_review(
+ &session_store,
+ &disabled_thread_store,
+ "review action",
+ /*extension_metrics*/ None
+ )
.await,
None
);
@@ -1309,7 +1430,12 @@ async fn contributor_reuses_the_latest_compatible_parent_compaction() -> Result<
assert_eq!(previous_score.scores.get("action_risk"), Some(&0.25));
assert_eq!(
registry
- .approval_review(&session_store, thread_store, "review action")
+ .approval_review(
+ &session_store,
+ thread_store,
+ "review action",
+ /*extension_metrics*/ None,
+ )
.await,
Some(ReviewDecision::Approved)
);
@@ -1348,7 +1474,12 @@ async fn contributor_reuses_the_latest_compatible_parent_compaction() -> Result<
);
assert_eq!(
registry
- .approval_review(&session_store, thread_store, "review action")
+ .approval_review(
+ &session_store,
+ thread_store,
+ "review action",
+ /*extension_metrics*/ None,
+ )
.await,
None
);
diff --git a/codex-rs/ext/skills/src/extension_tests.rs b/codex-rs/ext/skills/src/extension_tests.rs
index 109db2a91a..604ceaebf4 100644
--- a/codex-rs/ext/skills/src/extension_tests.rs
+++ b/codex-rs/ext/skills/src/extension_tests.rs
@@ -15,6 +15,10 @@ struct RecordingMetrics {
}
impl ExtensionMetrics for RecordingMetrics {
+ fn counter(&self, name: &str, _inc: i64, _tags: &[(&str, &str)]) {
+ panic!("unexpected counter: {name}");
+ }
+
fn histogram(&self, name: &str, value: i64, _tags: &[(&str, &str)]) {
self.samples
.lock()
diff --git a/codex-rs/ext/skills/src/render_observability_tests.rs b/codex-rs/ext/skills/src/render_observability_tests.rs
index 8e8f7383f7..a3715f5891 100644
--- a/codex-rs/ext/skills/src/render_observability_tests.rs
+++ b/codex-rs/ext/skills/src/render_observability_tests.rs
@@ -18,6 +18,10 @@ struct RecordingMetrics {
}
impl ExtensionMetrics for RecordingMetrics {
+ fn counter(&self, name: &str, _inc: i64, _tags: &[(&str, &str)]) {
+ panic!("unexpected counter: {name}");
+ }
+
fn histogram(&self, name: &str, value: i64, tags: &[(&str, &str)]) {
self.samples
.lock()
diff --git a/codex-rs/ext/skills/tests/skills_extension.rs b/codex-rs/ext/skills/tests/skills_extension.rs
index 257363a031..dd0a303064 100644
--- a/codex-rs/ext/skills/tests/skills_extension.rs
+++ b/codex-rs/ext/skills/tests/skills_extension.rs
@@ -2294,6 +2294,10 @@ impl RecordingMetrics {
}
impl ExtensionMetrics for RecordingMetrics {
+ fn counter(&self, name: &str, _inc: i64, _tags: &[(&str, &str)]) {
+ panic!("unexpected counter: {name}");
+ }
+
fn histogram(&self, name: &str, value: i64, tags: &[(&str, &str)]) {
self.samples
.lock()