Skip to content

armgit / codex

Public Imported from GitHub
0

Lightweight coding agent that runs in your terminal

Rust

Preserve specific MCP failure codes in metrics and spans (#54094)

Anthony Tafoya12fd621f6278
diff --git a/codex-rs/codex-mcp/src/binding.rs b/codex-rs/codex-mcp/src/binding.rs
index a20d1a8e35..e19001d8af 100644
--- a/codex-rs/codex-mcp/src/binding.rs
+++ b/codex-rs/codex-mcp/src/binding.rs
@@ -410,7 +410,7 @@ impl PreparedMcpCall {
tokio::time::timeout_at(deadline, add_trusted_access_context)
.await
.map_err(|_| {
- anyhow::anyhow!("timed out awaiting tools/call after {timeout:.0?}")
+ codex_rmcp_client::request_timeout_error("tools/call", timeout)
})?
}
None => add_trusted_access_context.await,
@@ -419,9 +419,7 @@ impl PreparedMcpCall {
Some((timeout, deadline)) => {
let remaining = deadline.saturating_duration_since(tokio::time::Instant::now());
if remaining.is_zero() {
- return Err(anyhow::anyhow!(
- "timed out awaiting tools/call after {timeout:.0?}"
- ));
+ return Err(codex_rmcp_client::request_timeout_error("tools/call", timeout));
}
Some(remaining)
}
diff --git a/codex-rs/codex-mcp/src/connection_manager_tests.rs b/codex-rs/codex-mcp/src/connection_manager_tests.rs
index 4e427d3a21..4bed4eebb0 100644
--- a/codex-rs/codex-mcp/src/connection_manager_tests.rs
+++ b/codex-rs/codex-mcp/src/connection_manager_tests.rs
@@ -548,6 +548,10 @@ async fn prepared_call_timeout_includes_trusted_access_lookup() {
assert_eq!(started.elapsed(), Duration::from_secs(1));
assert!(format!("{error:#}").contains("timed out awaiting tools/call after 1s"));
+ assert_eq!(
+ codex_rmcp_client::request_error_code(&error),
+ Some("request_timeout")
+ );
}
pub(crate) async fn create_ready_async_managed_client(tools: Vec<ToolInfo>) -> AsyncManagedClient {
diff --git a/codex-rs/core/src/mcp_tool_call.rs b/codex-rs/core/src/mcp_tool_call.rs
index 0cc6b1fc4a..c161ede8da 100644
--- a/codex-rs/core/src/mcp_tool_call.rs
+++ b/codex-rs/core/src/mcp_tool_call.rs
@@ -469,6 +469,7 @@ async fn handle_approved_mcp_tool_call(
.clone()
.unwrap_or_else(|| JsonValue::Object(serde_json::Map::new()));
let mut elicitation_type = None;
+ let mut request_error_code = None;
let result = async {
let result = async {
let mut result = prepared_call
@@ -551,7 +552,10 @@ async fn handle_approved_mcp_tool_call(
))
})
.await
- .map_err(|error| format!("tool call error: {error:?}"))?;
+ .map_err(|error| {
+ request_error_code = codex_rmcp_client::request_error_code(&error);
+ format!("tool call error: {error:?}")
+ })?;
crate::tools::record_confirmed_code_mode_send(
sess,
&turn_context.sub_id,
@@ -592,7 +596,7 @@ async fn handle_approved_mcp_tool_call(
.await)
}
.await;
- record_mcp_result_span_telemetry(&Span::current(), &result);
+ record_mcp_result_span_telemetry(&Span::current(), &result, request_error_code);
result
}
.instrument(mcp_tool_call_span(
@@ -642,7 +646,7 @@ async fn handle_approved_mcp_tool_call(
.await;
maybe_track_codex_app_used(sess, step_context, &server, &metadata, elicitation_type).await;
- let outcome = mcp_call_metric_outcome(&result);
+ let outcome = mcp_call_metric_outcome(&result).with_request_error_code(request_error_code);
emit_mcp_call_metrics(
&step_context.session_telemetry,
&outcome,
@@ -720,8 +724,13 @@ fn record_server_fields(span: &Span, url: Option<&str>) {
}
}
-fn record_mcp_result_span_telemetry(span: &Span, result: &Result<CallToolResult, String>) {
- record_mcp_call_outcome_span_telemetry(span, result);
+fn record_mcp_result_span_telemetry(
+ span: &Span,
+ result: &Result<CallToolResult, String>,
+ request_error_code: Option<&str>,
+) {
+ let outcome = mcp_call_metric_outcome(result).with_request_error_code(request_error_code);
+ record_mcp_call_outcome_span_telemetry(span, &outcome);
let Some(span_telemetry) = result
.as_ref()
diff --git a/codex-rs/core/src/mcp_tool_call/telemetry.rs b/codex-rs/core/src/mcp_tool_call/telemetry.rs
index 76aadd4ef6..dfa6feb095 100644
--- a/codex-rs/core/src/mcp_tool_call/telemetry.rs
+++ b/codex-rs/core/src/mcp_tool_call/telemetry.rs
@@ -31,6 +31,15 @@ pub(super) struct McpCallMetricOutcome {
}
impl McpCallMetricOutcome {
+ pub(super) fn with_request_error_code(mut self, code: Option<&str>) -> Self {
+ if self.error_type == Some(MCP_CALL_ERROR_TYPE_MCP_REQUEST)
+ && let Some(code) = code
+ {
+ self.error_code = Some(code.to_string());
+ }
+ self
+ }
+
pub(super) fn from_status(status: &'static str) -> Self {
Self {
status,
@@ -149,6 +158,15 @@ pub(super) fn mcp_call_metric_outcome(
.and_then(|auth_failure| auth_failure.get("error_code"))
.and_then(JsonValue::as_str)
.filter(|error_code| !error_code.is_empty())
+ })
+ .or_else(|| {
+ result
+ .meta
+ .as_ref()
+ .and_then(|meta| meta.get(super::MCP_RESULT_TELEMETRY_META_KEY))
+ .and_then(|telemetry| telemetry.get("error_code"))
+ .and_then(JsonValue::as_str)
+ .and_then(allowlisted_codex_telemetry_error_code)
});
let error_code: String = error_code
.unwrap_or(MCP_CALL_ERROR_CODE_UNKNOWN)
@@ -172,12 +190,32 @@ pub(super) fn mcp_call_metric_outcome(
}
}
-pub(super) fn record_mcp_call_outcome_span_telemetry(
- span: &Span,
- result: &Result<CallToolResult, String>,
-) {
- let outcome = mcp_call_metric_outcome(result);
- let (Some(error_type), Some(error_code)) = (outcome.error_type, outcome.error_code) else {
+fn allowlisted_codex_telemetry_error_code(code: &str) -> Option<&str> {
+ matches!(
+ code,
+ "kernel_start_failed"
+ | "kernel_exited"
+ | "kernel_io_error"
+ | "execution_timeout"
+ | "execution_cancelled"
+ | "execution_error"
+ | "execution_rejected"
+ | "execution_reset"
+ | "import_error"
+ | "syntax_error"
+ | "reference_error"
+ | "type_error"
+ | "uncaught_exception"
+ | "unhandled_rejection"
+ | "locator_timeout"
+ | "guardian_denied"
+ )
+ .then_some(code)
+}
+
+pub(super) fn record_mcp_call_outcome_span_telemetry(span: &Span, outcome: &McpCallMetricOutcome) {
+ let (Some(error_type), Some(error_code)) = (outcome.error_type, outcome.error_code.as_deref())
+ else {
return;
};
span.record(MCP_CALL_ERROR_TYPE_SPAN_ATTR, error_type);
diff --git a/codex-rs/core/src/mcp_tool_call/telemetry_tests.rs b/codex-rs/core/src/mcp_tool_call/telemetry_tests.rs
index 80b67e167d..d34085fc79 100644
--- a/codex-rs/core/src/mcp_tool_call/telemetry_tests.rs
+++ b/codex-rs/core/src/mcp_tool_call/telemetry_tests.rs
@@ -114,16 +114,101 @@ fn mcp_call_metric_outcome_reads_auth_error_code_from_meta() {
);
}
+#[test]
+fn mcp_call_metric_outcome_reads_telemetry_without_changing_model_output() {
+ let mut result = CallToolResult {
+ content: vec![
+ serde_json::json!({"type": "text", "text": "browser failed"}),
+ serde_json::json!({"type": "text", "text": "captured output"}),
+ ],
+ structured_content: None,
+ is_error: Some(true),
+ meta: None,
+ };
+ let original_output = result.as_function_call_output_payload();
+ result.meta = Some(serde_json::json!({
+ "codex/telemetry": {"error_code": "locator_timeout"},
+ }));
+
+ assert_eq!(result.as_function_call_output_payload(), original_output);
+ assert_eq!(
+ mcp_call_metric_outcome(&Ok(result)),
+ McpCallMetricOutcome {
+ status: "error",
+ error_type: Some(MCP_CALL_ERROR_TYPE_TOOL_RESULT),
+ error_code: Some("locator_timeout".to_string()),
+ approval_source: None,
+ }
+ );
+}
+
+#[test]
+fn mcp_call_metric_outcome_preserves_existing_error_code_precedence() {
+ let mut result = metric_call_tool_result(
+ /*is_error*/ true,
+ Some(serde_json::json!({"error_code": "RATE_LIMITED"})),
+ );
+ result.meta = Some(serde_json::json!({
+ "codex/telemetry": {"error_code": "locator_timeout"},
+ MCP_TOOL_CODEX_APPS_META_KEY: {
+ "connector_auth_failure": {
+ "is_auth_failure": true,
+ "error_code": "UNAUTHORIZED",
+ },
+ },
+ }));
+ for (structured_content, expected_code) in [
+ (result.structured_content.clone(), "RATE_LIMITED"),
+ (None, "UNAUTHORIZED"),
+ ] {
+ result.structured_content = structured_content;
+ assert_eq!(
+ mcp_call_metric_outcome(&Ok(result.clone())),
+ McpCallMetricOutcome {
+ status: "error",
+ error_type: Some(MCP_CALL_ERROR_TYPE_TOOL_RESULT),
+ error_code: Some(expected_code.to_string()),
+ approval_source: None,
+ }
+ );
+ }
+}
+
+#[test]
+fn mcp_call_metric_outcome_ignores_invalid_telemetry_codes() {
+ for metadata in [
+ serde_json::json!({"codex/telemetry": "invalid"}),
+ serde_json::json!({"codex/telemetry": {"error_code": 42}}),
+ serde_json::json!({"codex/telemetry": {"error_code": "Bearer credential-sentinel"}}),
+ serde_json::json!({"codex/telemetry": {"error_code": "locator_timeout:secret-sentinel"}}),
+ ] {
+ let mut result = metric_call_tool_result(
+ /*is_error*/ true,
+ Some(serde_json::json!({"error_code": ""})),
+ );
+ result.meta = Some(metadata);
+ assert_eq!(
+ mcp_call_metric_outcome(&Ok(result)),
+ McpCallMetricOutcome {
+ status: "error",
+ error_type: Some(MCP_CALL_ERROR_TYPE_TOOL_RESULT),
+ error_code: Some(MCP_CALL_ERROR_CODE_UNKNOWN.to_string()),
+ approval_source: None,
+ }
+ );
+ }
+}
+
#[test]
fn mcp_call_metric_outcome_bounds_and_sanitizes_error_code() {
let raw_error_code = format!("BAD CODE {}", "x".repeat(300));
- let result = Ok(metric_call_tool_result(
+ let result = metric_call_tool_result(
/*is_error*/ true,
Some(serde_json::json!({"error_code": raw_error_code})),
- ));
+ );
assert_eq!(
- mcp_call_metric_outcome(&result),
+ mcp_call_metric_outcome(&Ok(result)),
McpCallMetricOutcome {
status: "error",
error_type: Some(MCP_CALL_ERROR_TYPE_TOOL_RESULT),
@@ -132,3 +217,25 @@ fn mcp_call_metric_outcome_bounds_and_sanitizes_error_code() {
}
);
}
+
+#[test]
+fn caller_error_code_only_applies_to_request_failures() {
+ assert_eq!(
+ mcp_call_metric_outcome(&Err("deadline".to_string()))
+ .with_request_error_code(Some("request_timeout")),
+ McpCallMetricOutcome {
+ status: "error",
+ error_type: Some(MCP_CALL_ERROR_TYPE_MCP_REQUEST),
+ error_code: Some("request_timeout".to_string()),
+ approval_source: None,
+ }
+ );
+ let result = Ok(metric_call_tool_result(
+ /*is_error*/ true,
+ Some(serde_json::json!({"error_code": "locator_timeout"})),
+ ));
+ assert_eq!(
+ mcp_call_metric_outcome(&result).with_request_error_code(Some("request_timeout")),
+ mcp_call_metric_outcome(&result)
+ );
+}
diff --git a/codex-rs/core/src/mcp_tool_call_tests.rs b/codex-rs/core/src/mcp_tool_call_tests.rs
index e4b9dbfe73..b4145fee25 100644
--- a/codex-rs/core/src/mcp_tool_call_tests.rs
+++ b/codex-rs/core/src/mcp_tool_call_tests.rs
@@ -50,6 +50,9 @@ use core_test_support::responses::ev_response_created;
use core_test_support::responses::mount_sse_once;
use core_test_support::responses::sse;
use core_test_support::responses::start_mock_server;
+use opentelemetry::trace::TracerProvider;
+use opentelemetry_sdk::trace::InMemorySpanExporter;
+use opentelemetry_sdk::trace::SdkTracerProvider;
use pretty_assertions::assert_eq;
use serde::Deserialize;
use std::collections::HashMap;
@@ -58,6 +61,7 @@ use tempfile::tempdir;
use tracing::Instrument;
use tracing::Level;
use tracing_subscriber::fmt::format::FmtSpan;
+use tracing_subscriber::prelude::*;
use tracing_test::internal::MockWriter;
fn annotations(
@@ -500,26 +504,40 @@ async fn mcp_tool_call_span_records_expected_fields() {
);
}
+#[test_case::test_case(Some("RATE_LIMITED"), None, None, "RATE_LIMITED"; "structured error")]
+#[test_case::test_case(None, Some("credential-sentinel"), None, "unknown"; "sensitive metadata")]
+#[test_case::test_case(None, None, Some("transport_closed"), "transport_closed"; "closed transport")]
#[tokio::test]
-async fn mcp_tool_call_span_records_error_type_and_error_code() {
- let buffer: &'static std::sync::Mutex<Vec<u8>> =
- Box::leak(Box::new(std::sync::Mutex::new(Vec::new())));
- let subscriber = tracing_subscriber::fmt()
- .with_level(true)
- .with_ansi(false)
- .with_max_level(Level::TRACE)
- .with_span_events(FmtSpan::FULL)
- .with_writer(MockWriter::new(buffer))
- .finish();
+async fn mcp_tool_call_span_records_error_type_and_error_code(
+ structured_code: Option<&str>,
+ metadata_code: Option<&str>,
+ request_error_code: Option<&str>,
+ expected_code: &str,
+) {
+ let exporter = InMemorySpanExporter::default();
+ let provider = SdkTracerProvider::builder()
+ .with_simple_exporter(exporter.clone())
+ .build();
+ let subscriber = tracing_subscriber::registry()
+ .with(tracing_opentelemetry::layer().with_tracer(provider.tracer("mcp-error-test")));
let _guard = tracing::subscriber::set_default(subscriber);
let (session, turn_context) = make_session_and_context().await;
let result = Ok(CallToolResult {
content: Vec::new(),
- structured_content: Some(serde_json::json!({"error_code": "RATE_LIMITED"})),
+ structured_content: structured_code.map(|code| serde_json::json!({"error_code": code})),
is_error: Some(true),
- meta: None,
+ meta: metadata_code.map(|code| {
+ serde_json::json!({
+ "codex/telemetry": {"error_code": code},
+ })
+ }),
});
+ let result = if request_error_code.is_some() {
+ Err("request failed".to_string())
+ } else {
+ result
+ };
let span = mcp_tool_call_span(
&session,
&turn_context,
@@ -534,16 +552,42 @@ async fn mcp_tool_call_span_records_error_type_and_error_code() {
);
async {
- record_mcp_result_span_telemetry(&Span::current(), &result);
+ record_mcp_result_span_telemetry(&Span::current(), &result, request_error_code);
}
.instrument(span)
.await;
- let logs = String::from_utf8(buffer.lock().expect("buffer lock").clone()).expect("utf8 logs");
+ provider.force_flush().expect("flush spans");
+ let spans = exporter.get_finished_spans().expect("exported spans");
+ let span = spans
+ .iter()
+ .find(|span| span.name == "mcp.tools.call")
+ .expect("MCP tool span");
+ let error_codes = span
+ .attributes
+ .iter()
+ .filter(|attribute| attribute.key.as_str() == "codex.mcp.error.code")
+ .map(|attribute| attribute.value.to_string())
+ .collect::<Vec<_>>();
+ assert_eq!(error_codes, vec![expected_code.to_string()]);
+ let error_types = span
+ .attributes
+ .iter()
+ .filter(|attribute| attribute.key.as_str() == "error.type")
+ .map(|attribute| attribute.value.to_string())
+ .collect::<Vec<_>>();
+ assert_eq!(
+ error_types,
+ vec![if request_error_code.is_some() {
+ "mcp_request".to_string()
+ } else {
+ "tool_result".to_string()
+ }]
+ );
assert!(
- logs.contains("error.type=\"tool_result\"")
- && logs.contains("codex.mcp.error.code=\"RATE_LIMITED\""),
- "missing MCP tool error span fields\nlogs:\n{logs}"
+ span.attributes
+ .iter()
+ .all(|attribute| !attribute.value.to_string().contains("credential-sentinel"))
);
}
@@ -582,7 +626,11 @@ async fn mcp_result_telemetry_span_logs(meta: Option<serde_json::Value>) -> Stri
);
async {
- record_mcp_result_span_telemetry(&Span::current(), &result);
+ record_mcp_result_span_telemetry(
+ &Span::current(),
+ &result,
+ /*request_error_code*/ None,
+ );
}
.instrument(span)
.await;
diff --git a/codex-rs/rmcp-client/src/lib.rs b/codex-rs/rmcp-client/src/lib.rs
index a2f501171a..54b46ef07c 100644
--- a/codex-rs/rmcp-client/src/lib.rs
+++ b/codex-rs/rmcp-client/src/lib.rs
@@ -97,6 +97,8 @@ pub use rmcp_client::SendElicitation;
pub use rmcp_client::StreamableHttpBearerToken;
pub use rmcp_client::ToolWithConnectorId;
pub use service_error::mcp_error;
+pub use service_error::request_error_code;
+pub use service_error::request_timeout_error;
pub use startup_error::is_authentication_required_error;
pub use stdio_server_launcher::ExecutorStdioServerLauncher;
pub use stdio_server_launcher::LocalStdioServerLauncher;
diff --git a/codex-rs/rmcp-client/src/service_error.rs b/codex-rs/rmcp-client/src/service_error.rs
index 65da799f14..60dd3d2990 100644
--- a/codex-rs/rmcp-client/src/service_error.rs
+++ b/codex-rs/rmcp-client/src/service_error.rs
@@ -3,6 +3,7 @@
use anyhow::Error;
use rmcp::ErrorData;
use rmcp::service::ServiceError;
+use std::time::Duration;
use crate::rmcp_client::ClientOperationError;
@@ -13,3 +14,35 @@ pub fn mcp_error(error: &Error) -> Option<&ErrorData> {
};
Some(error)
}
+
+/// Preserves a request deadline as a typed error for caller-side telemetry.
+pub fn request_timeout_error(label: &str, duration: Duration) -> Error {
+ ClientOperationError::Timeout {
+ label: label.to_string(),
+ duration,
+ }
+ .into()
+}
+
+/// Returns a bounded caller-side failure category without parsing error text.
+pub fn request_error_code(error: &Error) -> Option<&'static str> {
+ let error = error.downcast_ref::<ClientOperationError>()?;
+ Some(match error {
+ ClientOperationError::Timeout { .. } => "request_timeout",
+ ClientOperationError::Service(error) => match error {
+ ServiceError::Timeout { .. } => "request_timeout",
+ ServiceError::TransportClosed => "transport_closed",
+ ServiceError::TransportSend(_) => "transport_error",
+ ServiceError::McpError(_) => "protocol_error",
+ ServiceError::UnexpectedResponse => "unexpected_response",
+ ServiceError::Cancelled { .. } => "cancelled",
+ ServiceError::SubscriptionLagged { .. } => "subscription_lagged",
+ ServiceError::InputRequiredRoundsExceeded { .. } => "input_required_rounds_exceeded",
+ _ => return None,
+ },
+ })
+}
+
+#[cfg(test)]
+#[path = "service_error_tests.rs"]
+mod tests;
diff --git a/codex-rs/rmcp-client/src/service_error_tests.rs b/codex-rs/rmcp-client/src/service_error_tests.rs
new file mode 100644
index 0000000000..692f070da1
--- /dev/null
+++ b/codex-rs/rmcp-client/src/service_error_tests.rs
@@ -0,0 +1,24 @@
+//! Verify caller error classification preserves typed causes through context.
+
+use super::*;
+use pretty_assertions::assert_eq;
+
+#[test]
+fn request_error_codes_preserve_typed_causes_through_context() {
+ for (error, expected) in [
+ (
+ request_timeout_error("tools/call", Duration::from_secs(1)),
+ Some("request_timeout"),
+ ),
+ (
+ anyhow::Error::from(ClientOperationError::Service(ServiceError::TransportClosed)),
+ Some("transport_closed"),
+ ),
+ (anyhow::anyhow!("timed out awaiting tools/call"), None),
+ ] {
+ assert_eq!(
+ request_error_code(&error.context("tool call failed")),
+ expected
+ );
+ }
+}