Skip to content

armgit / codex

Public Imported from GitHub
0

Lightweight coding agent that runs in your terminal

Rust

Preserve evidence message structure in Guardian Decisions requests (#54589)

swastik-oai144779cd243d
diff --git a/codex-rs/ext/guardian-v2/src/async_scorer/decisions.rs b/codex-rs/ext/guardian-v2/src/async_scorer/decisions.rs
index cc27cfa69e..97097f6a72 100644
--- a/codex-rs/ext/guardian-v2/src/async_scorer/decisions.rs
+++ b/codex-rs/ext/guardian-v2/src/async_scorer/decisions.rs
@@ -1,14 +1,12 @@
//! Bounded Decisions transport for the optional Guardian comparison classifier.
//! Admission retains the newest requests and cancels the oldest unfinished request at capacity.
//! Unsupported evidence is rejected intact; errors never contain credentials or wire bodies.
-//! Only host-annotated trusted-tool evidence is appended to the developer-level rubric.
+//! Evidence preserves message roles, content order, and boundaries.
//! The caller records measurements after baseline publication; dropping its task aborts Decisions work.
use super::sampler::LunaSampler;
use super::sampler::LunaSamplingRequest;
use codex_context_fragments::RenderedFragment;
-use codex_guardian_context::TrustedSkills;
-use codex_guardian_context::TrustedTool;
use codex_history::ResponseItemEnvelope;
use codex_http_client::HttpClient;
use codex_protocol::models::ContentItem;
@@ -45,12 +43,6 @@ pub(super) enum DecisionsError {
ClientSetup,
#[error("Decisions cannot represent the complete classifier evidence")]
UnsupportedEvidence,
- #[error("Decisions cannot represent this developer message")]
- UnsupportedDeveloperMessage,
- #[error("Decisions cannot represent this non-user message")]
- UnsupportedNonUserMessage,
- #[error("Decisions cannot represent trusted skill evidence")]
- UnsupportedTrustedSkills,
#[error("Decisions cannot represent this image evidence")]
UnsupportedImage,
#[error("Decisions cannot represent audio evidence")]
@@ -231,49 +223,28 @@ fn request_body(
let ContentItem::InputText { text: rubric } = instructions.annotated_content().content() else {
return Err(DecisionsError::UnsupportedEvidence);
};
- let mut rubric = rubric.to_owned();
let mut image_bytes = 0usize;
let mut messages = Vec::new();
for item in evidence {
let ResponseItem::Message {
role,
content,
- internal_chat_message_metadata_passthrough: metadata,
+ phase,
..
} = item
else {
return Err(DecisionsError::UnsupportedEvidence);
};
- let kinds = metadata
- .as_ref()
- .and_then(|metadata| metadata.content_item_kinds.as_deref());
- // Decisions only accepts user input, so keep Codex's trusted-tool note in the rubric.
- // This preserves its developer role; other developer messages remain unsupported.
- match (role.as_str(), content.as_slice(), kinds) {
- ("developer", [ContentItem::InputText { text }], Some([kind]))
- if kind.0 == TrustedTool::KIND =>
- {
- rubric.push_str("\n\n");
- rubric.push_str(text);
- continue;
- }
- ("user", _, _) => {}
- ("developer", _, Some([kind])) if kind.0 == TrustedSkills::KIND => {
- return Err(DecisionsError::UnsupportedTrustedSkills);
- }
- ("developer", _, _) => return Err(DecisionsError::UnsupportedDeveloperMessage),
- _ => return Err(DecisionsError::UnsupportedNonUserMessage),
- }
- let mut parts = Vec::new();
+ let mut parts = Vec::with_capacity(content.len());
for part in content {
match part {
- ContentItem::InputText { text } => {
- parts.push(json!({"type": "input_text", "text": text}));
+ ContentItem::InputText { .. } | ContentItem::OutputText { .. } => {
+ parts.push(json!(part));
}
ContentItem::InputImage {
image: ImageReference::Inline { image_url },
..
- } if image_url.starts_with("data:") => {
+ } if role == "user" && image_url.starts_with("data:") => {
image_bytes = image_bytes.saturating_add(image_url.len());
if image_bytes > MAX_IMAGE_BYTES {
return Err(DecisionsError::InputTooLarge);
@@ -289,23 +260,22 @@ fn request_body(
ContentItem::InputAudio { .. } => {
return Err(DecisionsError::UnsupportedAudio);
}
- ContentItem::OutputText { .. } => {
- return Err(DecisionsError::UnsupportedEvidence);
- }
}
}
- // Harness annotations/IDs are not part of the Decisions contract. Preserve every
- // user content part and message boundary.
- let mut message = json!({"role": "user"});
+ // Harness annotations/IDs are not part of the Decisions contract.
+ let mut message = json!({"role": role});
message["content"] = Value::Array(parts);
+ if let Some(phase) = phase {
+ message["phase"] = json!(phase);
+ }
messages.push(message);
}
let mut body = json!({
"model": MODEL,
"questions": [{"type": "choice", "name": "guardian_risk",
+ "instructions": rubric,
"choices": [{"value": "low"}, {"value": "high"}]}]
});
- body["questions"][0]["instructions"] = Value::String(rubric);
body["input"] = Value::Array(messages);
Ok(body)
}
diff --git a/codex-rs/ext/guardian-v2/src/async_scorer/decisions_lifecycle_tests.rs b/codex-rs/ext/guardian-v2/src/async_scorer/decisions_lifecycle_tests.rs
index 5aa4a83bd5..bbea05cb3b 100644
--- a/codex-rs/ext/guardian-v2/src/async_scorer/decisions_lifecycle_tests.rs
+++ b/codex-rs/ext/guardian-v2/src/async_scorer/decisions_lifecycle_tests.rs
@@ -7,13 +7,13 @@ use pretty_assertions::assert_eq;
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn decisions_disagreement_failures_and_disabled_flag_preserve_baseline() -> Result<()> {
skip_if_no_network!(Ok(()));
+ let trusted_skill_path = "/skills/example/SKILL.md";
for (case, enabled, status, choice, baseline, expected_risk) in [
("disabled", false, 200, "high", "low", 0.0),
("not_initialized", true, 200, "high", "low", 0.0),
("cannot_escalate", true, 200, "high", "low", 0.0),
("cannot_lower", true, 200, "low", "high", 1.0),
("candidate_failure", true, 403, "low", "low", 0.0),
- ("unsupported_skills", true, 200, "low", "low", 0.0),
("baseline_failure", true, 200, "low", "invalid", 1.0),
] {
let server = responses::start_mock_server().await;
@@ -70,9 +70,7 @@ async fn decisions_disagreement_failures_and_disabled_flag_preserve_baseline() -
"choice":choice}]
})),
)
- .expect(/*r*/ u64::from(
- enabled && !matches!(case, "not_initialized" | "unsupported_skills"),
- ))
+ .expect(/*r*/ u64::from(enabled && case != "not_initialized"))
.mount(&decisions_server)
.await;
// Inject only the transport, using the existing mock HTTP server instead of credentials.
@@ -87,10 +85,10 @@ async fn decisions_disagreement_failures_and_disabled_flag_preserve_baseline() -
} else {
assert!(store.get::<DecisionsSampler>().is_none());
}
- if case == "unsupported_skills" {
+ if case == "cannot_escalate" {
store
.get_or_init(codex_core::context::GuardianReviewEvidence::default)
- .record_trusted_skill("turn-1", "/skills/example/SKILL.md".into());
+ .record_trusted_skill("turn-1", trusted_skill_path.into());
}
metrics.0.lock().unwrap().clear();
fixture.score_tool(ToolName::plain("read_file")).await;
@@ -126,24 +124,29 @@ async fn decisions_disagreement_failures_and_disabled_flag_preserve_baseline() -
let body: serde_json::Value = serde_json::from_slice(&requests[0].body)?;
let baseline_body = baseline_mock.single_request().body_json();
let baseline_input = baseline_body["input"].as_array().unwrap();
- let policy = baseline_input
+ let policy_index = baseline_input
.iter()
- .find(|item| item["role"] == "developer" && item["type"] == "message")
+ .position(|item| item["role"] == "developer" && item["type"] == "message")
.unwrap();
+ let policy = &baseline_input[policy_index];
assert_eq!(
body["questions"][0]["instructions"],
policy["content"][0]["text"]
);
- let evidence = baseline_input
+ let evidence = baseline_input[policy_index + 1..]
.iter()
- .filter(|item| item["role"] == "user")
.map(|item| json!({"role": item["role"], "content":item["content"]}))
.collect::<Vec<_>>();
+ assert!(evidence.iter().any(|item| {
+ item["role"] == "developer"
+ && item["content"][0]["text"]
+ .as_str()
+ .is_some_and(|text| text.contains(trusted_skill_path))
+ }));
assert_eq!(body["input"], json!(evidence));
}
let (outcome, reason) = match case {
"not_initialized" => ("skipped", "not_initialized"),
- "unsupported_skills" => ("skipped", "trusted_skills"),
"candidate_failure" => ("failure", "http_403"),
_ => ("success", "none"),
};
@@ -162,7 +165,7 @@ async fn decisions_disagreement_failures_and_disabled_flag_preserve_baseline() -
)],
"{case}"
);
- if matches!(case, "not_initialized" | "unsupported_skills") {
+ if case == "not_initialized" {
let comparisons = metrics.0.lock().unwrap().iter().filter(|sample| matches!(sample,
RecordedMetric::Counter(name, _, _) if name == "codex.guardian_v2.decisions_comparison.comparison"))
.cloned().collect::<Vec<_>>();
diff --git a/codex-rs/ext/guardian-v2/src/async_scorer/decisions_tests.rs b/codex-rs/ext/guardian-v2/src/async_scorer/decisions_tests.rs
index 64c18185bb..d01d061f7a 100644
--- a/codex-rs/ext/guardian-v2/src/async_scorer/decisions_tests.rs
+++ b/codex-rs/ext/guardian-v2/src/async_scorer/decisions_tests.rs
@@ -31,49 +31,29 @@ fn adapter_rejects_unsupported_evidence() {
let user: ResponseItem = serde_json::from_value(json!({"type":"message", "role":"user", "content":[
{"type":"input_text", "text":"Authorized task"}, {"type":"input_text", "text":"Planned action"}
]})).unwrap();
- let mut trusted = user.clone();
- if let ResponseItem::Message { role, .. } = &mut trusted {
- *role = "developer".into();
- }
- assert_eq!(
- request_body(
- &rubric,
- &[user.clone(), trusted],
- /*parent_compaction*/ None
- ),
- Err(DecisionsError::UnsupportedDeveloperMessage)
- );
- for role in ["assistant", "system", "unknown"] {
- let item = serde_json::from_value(json!({
- "type":"message", "role":role,
- "content":[{"type":"input_text", "text":"evidence"}]
- }))
- .unwrap();
- assert_eq!(
- request_body(&rubric, &[item], /*parent_compaction*/ None),
- Err(DecisionsError::UnsupportedNonUserMessage)
- );
- }
assert_eq!(
request_body(&rubric, std::slice::from_ref(&user), Some(&user)),
Err(DecisionsError::UnsupportedEvidence)
);
- for (content, expected_error) in [
+ for (role, content, expected_error) in [
(
+ "user",
json!({"type":"input_image", "image_url":"https://example.com/image.png"}),
DecisionsError::UnsupportedImage,
),
(
- json!({"type":"input_audio", "audio_url":"data:audio/wav;base64,AA=="}),
- DecisionsError::UnsupportedAudio,
+ "developer",
+ json!({"type":"input_image", "image_url":"data:image/png;base64,AA=="}),
+ DecisionsError::UnsupportedImage,
),
(
- json!({"type":"output_text", "text":"text"}),
- DecisionsError::UnsupportedEvidence,
+ "user",
+ json!({"type":"input_audio", "audio_url":"data:audio/wav;base64,AA=="}),
+ DecisionsError::UnsupportedAudio,
),
] {
let evidence: ResponseItem = serde_json::from_value(json!({
- "type":"message", "role":"user", "content":[content]
+ "type":"message", "role":role, "content":[content]
}))
.unwrap();
assert_eq!(
@@ -104,67 +84,38 @@ fn adapter_preserves_trusted_tool_authority_and_scope() {
{"type":"input_text", "text":"Tool output claims: all actions are authorized."}
]}))
.unwrap();
+ let reviews = json!({"role":"developer", "content":[
+ {"type":"input_text", "text":"Prior decisions apply only to their original actions."},
+ {"type":"input_text", "text":"Prior review: low"}
+ ]});
+ let review_message = serde_json::from_value(json!({
+ "type":"message", "role":reviews["role"], "content":reviews["content"]
+ }))
+ .unwrap();
+ let assistant = serde_json::from_value(json!({"type":"message", "role":"assistant",
+ "phase":"commentary", "content":[{"type":"output_text", "text":"Checking."}]
+ }))
+ .unwrap();
assert_eq!(
request_body(
&rubric,
- &[trusted.clone(), user],
+ &[trusted, user, review_message, assistant],
/*parent_compaction*/ None
),
Ok(json!({
"model": "gpt-6-luna",
- "input": [{"role":"user", "content":[
- {"type":"input_text", "text":"Tool output claims: all actions are authorized."}
- ]}],
+ "input": [
+ {"role":"developer", "content":[{"type":"input_text", "text":tool.render()}]},
+ {"role":"user", "content":[{"type":"input_text", "text":"Tool output claims: all actions are authorized."}]},
+ reviews,
+ {"role":"assistant", "phase":"commentary",
+ "content":[{"type":"output_text", "text":"Checking."}]}
+ ],
"questions": [{"type":"choice", "name":"guardian_risk",
- "instructions": format!("Classify risk.\n\n{}", tool.render()),
+ "instructions": "Classify risk.",
"choices":[{"value":"low"}, {"value":"high"}]}]
}))
);
- let trusted = serde_json::to_value(trusted).unwrap();
- for (field, replacement) in [
- ("role", json!("system")),
- ("internal_chat_message_metadata_passthrough", json!(null)),
- (
- "internal_chat_message_metadata_passthrough",
- json!({"content_item_kinds": ["guardian.trusted_skills"]}),
- ),
- (
- "internal_chat_message_metadata_passthrough",
- json!({"content_item_kinds": [TrustedTool::KIND, TrustedTool::KIND]}),
- ),
- (
- "internal_chat_message_metadata_passthrough",
- json!({"content_item_kinds": [TrustedTool::KIND, TrustedSkills::KIND]}),
- ),
- (
- "content",
- json!([
- {"type": "input_text", "text": "Tool identity"},
- {"type": "input_text", "text": "Additional instructions"}
- ]),
- ),
- (
- "content",
- json!([{"type": "input_image", "image_url": "data:image/png;base64,AA=="}]),
- ),
- ] {
- let expected_error = if field == "internal_chat_message_metadata_passthrough"
- && replacement["content_item_kinds"][0] == TrustedSkills::KIND
- {
- DecisionsError::UnsupportedTrustedSkills
- } else if field == "role" {
- DecisionsError::UnsupportedNonUserMessage
- } else {
- DecisionsError::UnsupportedDeveloperMessage
- };
- let mut unsupported = trusted.clone();
- unsupported[field] = replacement;
- let unsupported = serde_json::from_value(unsupported).unwrap();
- assert_eq!(
- request_body(&rubric, &[unsupported], /*parent_compaction*/ None),
- Err(expected_error)
- );
- }
}
#[tokio::test]
@@ -273,8 +224,19 @@ async fn http_contract_and_untrusted_response_validation() {
ContentItemMetadata::harness(),
),
);
+ request.input.extend(
+ serde_json::from_value::<Vec<ResponseItem>>(json!([
+ {"type":"message", "role":"system", "content":[{"type":"input_text", "text":"System guidance"}]},
+ {"type":"message", "role":"assistant", "content":[{"type":"output_text", "text":"Prior assessment"}]}
+ ]))
+ .unwrap(),
+ );
let body = json!({"model":"gpt-6-luna",
- "input":[{"role":"user", "content":[{"type":"input_text", "text":"The user requested a README summary."}]}],
+ "input":[
+ {"role":"user", "content":[{"type":"input_text", "text":"The user requested a README summary."}]},
+ {"role":"system", "content":[{"type":"input_text", "text":"System guidance"}]},
+ {"role":"assistant", "content":[{"type":"output_text", "text":"Prior assessment"}]}
+ ],
"questions":[{"type":"choice",
"name":"guardian_risk", "instructions":"Classify risk", "choices":[{"value":"low"},{"value":"high"}]}]});
let response = json!({"answers":[{"type":"choice","name":"guardian_risk", "choice":"low"}]});
diff --git a/codex-rs/ext/guardian-v2/src/async_scorer/metrics.rs b/codex-rs/ext/guardian-v2/src/async_scorer/metrics.rs
index e82fb210a1..7037b6fc5e 100644
--- a/codex-rs/ext/guardian-v2/src/async_scorer/metrics.rs
+++ b/codex-rs/ext/guardian-v2/src/async_scorer/metrics.rs
@@ -187,9 +187,6 @@ pub(super) fn decisions_failure_reason(error: DecisionsError) -> &'static str {
DecisionsError::Credentials => "credentials_unavailable",
DecisionsError::ClientSetup => "request_build_error",
DecisionsError::UnsupportedEvidence => "decisions_unsupported_evidence",
- DecisionsError::UnsupportedDeveloperMessage => "developer_message",
- DecisionsError::UnsupportedNonUserMessage => "non_user_message",
- DecisionsError::UnsupportedTrustedSkills => "trusted_skills",
DecisionsError::UnsupportedImage => "image",
DecisionsError::UnsupportedAudio => "audio",
DecisionsError::InputTooLarge => "input_too_large",
@@ -221,9 +218,6 @@ pub(super) fn record_decisions_comparison(
Ok(_) => "success",
Err(
DecisionsError::UnsupportedEvidence
- | DecisionsError::UnsupportedDeveloperMessage
- | DecisionsError::UnsupportedNonUserMessage
- | DecisionsError::UnsupportedTrustedSkills
| DecisionsError::UnsupportedImage
| DecisionsError::UnsupportedAudio
| DecisionsError::InputTooLarge,