Skip to content

armgit / codex

Public Imported from GitHub
0

Lightweight coding agent that runs in your terminal

Rust

Distinguish unsupported evidence in Decisions comparison metrics (#54057)

swastik-oai1f7fe9b0afb3
diff --git a/codex-rs/ext/guardian-v2/src/async_scorer/decisions.rs b/codex-rs/ext/guardian-v2/src/async_scorer/decisions.rs
index eebaa04b7f..cc27cfa69e 100644
--- a/codex-rs/ext/guardian-v2/src/async_scorer/decisions.rs
+++ b/codex-rs/ext/guardian-v2/src/async_scorer/decisions.rs
@@ -7,6 +7,7 @@
use super::sampler::LunaSampler;
use super::sampler::LunaSamplingRequest;
use codex_context_fragments::RenderedFragment;
+use codex_guardian_context::TrustedSkills;
use codex_guardian_context::TrustedTool;
use codex_history::ResponseItemEnvelope;
use codex_http_client::HttpClient;
@@ -44,6 +45,16 @@ pub(super) enum DecisionsError {
ClientSetup,
#[error("Decisions cannot represent the complete classifier evidence")]
UnsupportedEvidence,
+ #[error("Decisions cannot represent this developer message")]
+ UnsupportedDeveloperMessage,
+ #[error("Decisions cannot represent this non-user message")]
+ UnsupportedNonUserMessage,
+ #[error("Decisions cannot represent trusted skill evidence")]
+ UnsupportedTrustedSkills,
+ #[error("Decisions cannot represent this image evidence")]
+ UnsupportedImage,
+ #[error("Decisions cannot represent audio evidence")]
+ UnsupportedAudio,
#[error("Decisions input exceeds the request limit")]
InputTooLarge,
#[error("Decisions request timed out")]
@@ -247,7 +258,11 @@ fn request_body(
continue;
}
("user", _, _) => {}
- _ => return Err(DecisionsError::UnsupportedEvidence),
+ ("developer", _, Some([kind])) if kind.0 == TrustedSkills::KIND => {
+ return Err(DecisionsError::UnsupportedTrustedSkills);
+ }
+ ("developer", _, _) => return Err(DecisionsError::UnsupportedDeveloperMessage),
+ _ => return Err(DecisionsError::UnsupportedNonUserMessage),
}
let mut parts = Vec::new();
for part in content {
@@ -268,9 +283,13 @@ fn request_body(
json!({"type": "input_image", "image_url": image_url, "detail": null}),
);
}
- ContentItem::InputImage { .. }
- | ContentItem::InputAudio { .. }
- | ContentItem::OutputText { .. } => {
+ ContentItem::InputImage { .. } => {
+ return Err(DecisionsError::UnsupportedImage);
+ }
+ ContentItem::InputAudio { .. } => {
+ return Err(DecisionsError::UnsupportedAudio);
+ }
+ ContentItem::OutputText { .. } => {
return Err(DecisionsError::UnsupportedEvidence);
}
}
diff --git a/codex-rs/ext/guardian-v2/src/async_scorer/decisions_lifecycle_tests.rs b/codex-rs/ext/guardian-v2/src/async_scorer/decisions_lifecycle_tests.rs
index 530381a7e0..5aa4a83bd5 100644
--- a/codex-rs/ext/guardian-v2/src/async_scorer/decisions_lifecycle_tests.rs
+++ b/codex-rs/ext/guardian-v2/src/async_scorer/decisions_lifecycle_tests.rs
@@ -13,6 +13,7 @@ async fn decisions_disagreement_failures_and_disabled_flag_preserve_baseline() -
("cannot_escalate", true, 200, "high", "low", 0.0),
("cannot_lower", true, 200, "low", "high", 1.0),
("candidate_failure", true, 403, "low", "low", 0.0),
+ ("unsupported_skills", true, 200, "low", "low", 0.0),
("baseline_failure", true, 200, "low", "invalid", 1.0),
] {
let server = responses::start_mock_server().await;
@@ -69,7 +70,9 @@ async fn decisions_disagreement_failures_and_disabled_flag_preserve_baseline() -
"choice":choice}]
})),
)
- .expect(/*r*/ u64::from(enabled && case != "not_initialized"))
+ .expect(/*r*/ u64::from(
+ enabled && !matches!(case, "not_initialized" | "unsupported_skills"),
+ ))
.mount(&decisions_server)
.await;
// Inject only the transport, using the existing mock HTTP server instead of credentials.
@@ -84,6 +87,11 @@ async fn decisions_disagreement_failures_and_disabled_flag_preserve_baseline() -
} else {
assert!(store.get::<DecisionsSampler>().is_none());
}
+ if case == "unsupported_skills" {
+ store
+ .get_or_init(codex_core::context::GuardianReviewEvidence::default)
+ .record_trusted_skill("turn-1", "/skills/example/SKILL.md".into());
+ }
metrics.0.lock().unwrap().clear();
fixture.score_tool(ToolName::plain("read_file")).await;
let score = tokio::time::timeout(ASYNC_TEST_TIMEOUT, async {
@@ -135,6 +143,7 @@ async fn decisions_disagreement_failures_and_disabled_flag_preserve_baseline() -
}
let (outcome, reason) = match case {
"not_initialized" => ("skipped", "not_initialized"),
+ "unsupported_skills" => ("skipped", "trusted_skills"),
"candidate_failure" => ("failure", "http_403"),
_ => ("success", "none"),
};
@@ -153,7 +162,7 @@ async fn decisions_disagreement_failures_and_disabled_flag_preserve_baseline() -
)],
"{case}"
);
- if case == "not_initialized" {
+ if matches!(case, "not_initialized" | "unsupported_skills") {
let comparisons = metrics.0.lock().unwrap().iter().filter(|sample| matches!(sample,
RecordedMetric::Counter(name, _, _) if name == "codex.guardian_v2.decisions_comparison.comparison"))
.cloned().collect::<Vec<_>>();
diff --git a/codex-rs/ext/guardian-v2/src/async_scorer/decisions_tests.rs b/codex-rs/ext/guardian-v2/src/async_scorer/decisions_tests.rs
index ddcd6f154b..64c18185bb 100644
--- a/codex-rs/ext/guardian-v2/src/async_scorer/decisions_tests.rs
+++ b/codex-rs/ext/guardian-v2/src/async_scorer/decisions_tests.rs
@@ -41,21 +41,46 @@ fn adapter_rejects_unsupported_evidence() {
&[user.clone(), trusted],
/*parent_compaction*/ None
),
- Err(DecisionsError::UnsupportedEvidence)
+ Err(DecisionsError::UnsupportedDeveloperMessage)
);
+ for role in ["assistant", "system", "unknown"] {
+ let item = serde_json::from_value(json!({
+ "type":"message", "role":role,
+ "content":[{"type":"input_text", "text":"evidence"}]
+ }))
+ .unwrap();
+ assert_eq!(
+ request_body(&rubric, &[item], /*parent_compaction*/ None),
+ Err(DecisionsError::UnsupportedNonUserMessage)
+ );
+ }
assert_eq!(
request_body(&rubric, std::slice::from_ref(&user), Some(&user)),
Err(DecisionsError::UnsupportedEvidence)
);
- let image: ResponseItem =
- serde_json::from_value(json!({"type":"message", "role":"user", "content":[
- {"type":"input_image", "image_url":"https://example.com/image.png"}
- ]}))
+ for (content, expected_error) in [
+ (
+ json!({"type":"input_image", "image_url":"https://example.com/image.png"}),
+ DecisionsError::UnsupportedImage,
+ ),
+ (
+ json!({"type":"input_audio", "audio_url":"data:audio/wav;base64,AA=="}),
+ DecisionsError::UnsupportedAudio,
+ ),
+ (
+ json!({"type":"output_text", "text":"text"}),
+ DecisionsError::UnsupportedEvidence,
+ ),
+ ] {
+ let evidence: ResponseItem = serde_json::from_value(json!({
+ "type":"message", "role":"user", "content":[content]
+ }))
.unwrap();
- assert_eq!(
- request_body(&rubric, &[image], /*parent_compaction*/ None),
- Err(DecisionsError::UnsupportedEvidence)
- );
+ assert_eq!(
+ request_body(&rubric, &[evidence], /*parent_compaction*/ None),
+ Err(expected_error)
+ );
+ }
}
#[test]
@@ -107,6 +132,10 @@ fn adapter_preserves_trusted_tool_authority_and_scope() {
"internal_chat_message_metadata_passthrough",
json!({"content_item_kinds": [TrustedTool::KIND, TrustedTool::KIND]}),
),
+ (
+ "internal_chat_message_metadata_passthrough",
+ json!({"content_item_kinds": [TrustedTool::KIND, TrustedSkills::KIND]}),
+ ),
(
"content",
json!([
@@ -119,12 +148,21 @@ fn adapter_preserves_trusted_tool_authority_and_scope() {
json!([{"type": "input_image", "image_url": "data:image/png;base64,AA=="}]),
),
] {
+ let expected_error = if field == "internal_chat_message_metadata_passthrough"
+ && replacement["content_item_kinds"][0] == TrustedSkills::KIND
+ {
+ DecisionsError::UnsupportedTrustedSkills
+ } else if field == "role" {
+ DecisionsError::UnsupportedNonUserMessage
+ } else {
+ DecisionsError::UnsupportedDeveloperMessage
+ };
let mut unsupported = trusted.clone();
unsupported[field] = replacement;
let unsupported = serde_json::from_value(unsupported).unwrap();
assert_eq!(
request_body(&rubric, &[unsupported], /*parent_compaction*/ None),
- Err(DecisionsError::UnsupportedEvidence)
+ Err(expected_error)
);
}
}
diff --git a/codex-rs/ext/guardian-v2/src/async_scorer/metrics.rs b/codex-rs/ext/guardian-v2/src/async_scorer/metrics.rs
index a2f2de215f..e82fb210a1 100644
--- a/codex-rs/ext/guardian-v2/src/async_scorer/metrics.rs
+++ b/codex-rs/ext/guardian-v2/src/async_scorer/metrics.rs
@@ -187,6 +187,11 @@ pub(super) fn decisions_failure_reason(error: DecisionsError) -> &'static str {
DecisionsError::Credentials => "credentials_unavailable",
DecisionsError::ClientSetup => "request_build_error",
DecisionsError::UnsupportedEvidence => "decisions_unsupported_evidence",
+ DecisionsError::UnsupportedDeveloperMessage => "developer_message",
+ DecisionsError::UnsupportedNonUserMessage => "non_user_message",
+ DecisionsError::UnsupportedTrustedSkills => "trusted_skills",
+ DecisionsError::UnsupportedImage => "image",
+ DecisionsError::UnsupportedAudio => "audio",
DecisionsError::InputTooLarge => "input_too_large",
DecisionsError::Timeout => "transport_timeout",
// The adapter erases transport details; do not claim a specific network failure.
@@ -214,9 +219,15 @@ pub(super) fn record_decisions_comparison(
Ok((result, duration)) => {
let outcome = match &result {
Ok(_) => "success",
- Err(DecisionsError::UnsupportedEvidence | DecisionsError::InputTooLarge) => {
- "skipped"
- }
+ Err(
+ DecisionsError::UnsupportedEvidence
+ | DecisionsError::UnsupportedDeveloperMessage
+ | DecisionsError::UnsupportedNonUserMessage
+ | DecisionsError::UnsupportedTrustedSkills
+ | DecisionsError::UnsupportedImage
+ | DecisionsError::UnsupportedAudio
+ | DecisionsError::InputTooLarge,
+ ) => "skipped",
Err(_) => "failure",
};
record_decisions_comparison_outcome(
diff --git a/codex-rs/guardian-context/src/trusted_skills.rs b/codex-rs/guardian-context/src/trusted_skills.rs
index 0907a7e177..0cbf56695d 100644
--- a/codex-rs/guardian-context/src/trusted_skills.rs
+++ b/codex-rs/guardian-context/src/trusted_skills.rs
@@ -20,9 +20,14 @@ pub struct TrustedSkills {
pub paths: Vec<String>,
}
+impl TrustedSkills {
+ /// Classification attached to this fragment's rendered text.
+ pub const KIND: &str = "guardian.trusted_skills";
+}
+
impl ContextualUserFragment for TrustedSkills {
fn content_kind(&self) -> ContentItemKind {
- ContentItemKind("guardian.trusted_skills".to_owned())
+ ContentItemKind(Self::KIND.to_owned())
}
fn role(&self) -> &'static str {