Skip to content

armgit / codex

Public Imported from GitHub
0

Lightweight coding agent that runs in your terminal

Rust

Honor Retry-After and preserve server retry deadlines (#47641)

Adam Perry @ OpenAI9d8de196748b
diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock
index dd28581942..5b74aa2320 100644
--- a/codex-rs/Cargo.lock
+++ b/codex-rs/Cargo.lock
@@ -3612,6 +3612,7 @@ dependencies = [
"futures",
"http 1.4.0",
"http-body 1.0.1",
+ "httpdate",
"native-tls",
"opentelemetry",
"opentelemetry_sdk",
diff --git a/codex-rs/cli/src/doctor.rs b/codex-rs/cli/src/doctor.rs
index 6cc337a881..8af977effb 100644
--- a/codex-rs/cli/src/doctor.rs
+++ b/codex-rs/cli/src/doctor.rs
@@ -2440,7 +2440,7 @@ fn websocket_error_detail(err: &ApiError) -> String {
| ApiError::CyberPolicy { .. }
| ApiError::BioPolicy { .. }
| ApiError::MisalignmentPolicyViolation { .. }
- | ApiError::ServerOverloaded => format!("handshake error: {err}"),
+ | ApiError::ServerOverloaded { .. } => format!("handshake error: {err}"),
}
}
diff --git a/codex-rs/codex-api/Cargo.toml b/codex-rs/codex-api/Cargo.toml
index da3f1b7fcb..82bec69218 100644
--- a/codex-rs/codex-api/Cargo.toml
+++ b/codex-rs/codex-api/Cargo.toml
@@ -37,6 +37,7 @@ pretty_assertions = { workspace = true }
rcgen = { workspace = true }
rustls = { workspace = true }
tempfile = { workspace = true }
+tokio = { workspace = true, features = ["test-util"] }
tokio-test = { workspace = true }
wiremock = { workspace = true }
diff --git a/codex-rs/codex-api/src/api_bridge.rs b/codex-rs/codex-api/src/api_bridge.rs
index a2cda1566e..2f6fe9a827 100644
--- a/codex-rs/codex-api/src/api_bridge.rs
+++ b/codex-rs/codex-api/src/api_bridge.rs
@@ -19,26 +19,42 @@ use serde::Deserialize;
use serde_json::Value;
pub fn map_api_error(err: ApiError) -> CodexErr {
+ let retry_after = match &err {
+ ApiError::Retryable { retry_after, .. }
+ | ApiError::RateLimitExceeded { retry_after, .. }
+ | ApiError::ServerOverloaded { retry_after }
+ | ApiError::Transport(TransportError::Http { retry_after, .. }) => *retry_after,
+ ApiError::Transport(_)
+ | ApiError::Api { .. }
+ | ApiError::Stream(_)
+ | ApiError::ContextWindowExceeded
+ | ApiError::QuotaExceeded
+ | ApiError::UsageNotIncluded
+ | ApiError::RateLimit(_)
+ | ApiError::InvalidRequest { .. }
+ | ApiError::InvalidPrompt { .. }
+ | ApiError::CyberPolicy { .. }
+ | ApiError::BioPolicy { .. }
+ | ApiError::MisalignmentPolicyViolation { .. } => None,
+ };
+ let error = map_api_error_details(err);
+ match retry_after {
+ Some(retry_after) => error.with_retry_after(retry_after),
+ None => error,
+ }
+}
+
+fn map_api_error_details(err: ApiError) -> CodexErr {
match err {
ApiError::ContextWindowExceeded => CodexErr::ContextWindowExceeded,
ApiError::QuotaExceeded => CodexErr::QuotaExceeded,
ApiError::UsageNotIncluded => CodexErr::UsageNotIncluded,
- ApiError::Retryable { message, delay } => {
- let error = CodexErr::Stream(message);
- match delay {
- Some(delay) => error.with_retry_delay(delay),
- None => error,
- }
- }
- ApiError::RateLimitExceeded { message, delay } => {
- let error = CodexErr::new(CodexErrorDetails::RateLimitExceeded(message));
- match delay {
- Some(delay) => error.with_retry_delay(delay),
- None => error,
- }
+ ApiError::Retryable { message, .. } => CodexErr::Stream(message),
+ ApiError::RateLimitExceeded { message, .. } => {
+ CodexErr::new(CodexErrorDetails::RateLimitExceeded(message))
}
ApiError::Stream(msg) => CodexErr::Stream(msg),
- ApiError::ServerOverloaded => CodexErr::ServerOverloaded,
+ ApiError::ServerOverloaded { .. } => CodexErr::ServerOverloaded,
ApiError::Api { status, message } => {
let user_message = api_error_user_message(status, &message);
CodexErr::UnexpectedStatus(UnexpectedResponseError {
@@ -73,6 +89,7 @@ pub fn map_api_error(err: ApiError) -> CodexErr {
url,
headers,
body,
+ ..
} => {
let body_text = body.unwrap_or_default();
diff --git a/codex-rs/codex-api/src/api_bridge_tests.rs b/codex-rs/codex-api/src/api_bridge_tests.rs
index 660262ea97..b7ed61ccf5 100644
--- a/codex-rs/codex-api/src/api_bridge_tests.rs
+++ b/codex-rs/codex-api/src/api_bridge_tests.rs
@@ -1,23 +1,25 @@
use super::*;
use base64::Engine;
+use codex_http_client::RetryAfter;
use codex_protocol::protocol::CodexErrorInfo;
use codex_protocol::protocol::RateLimitReachedType;
use pretty_assertions::assert_eq;
#[test]
fn map_api_error_maps_server_overloaded() {
- let err = map_api_error(ApiError::ServerOverloaded);
+ let err = map_api_error(ApiError::ServerOverloaded { retry_after: None });
assert!(matches!(err.details(), CodexErrorDetails::ServerOverloaded));
}
-#[test]
-fn map_api_error_preserves_retry_delay() {
+#[tokio::test(start_paused = true)]
+async fn map_api_error_preserves_retry_delay() {
let retry_delay = std::time::Duration::from_secs(17);
+ let retry_after = RetryAfter::from_delay(retry_delay).expect("retry advice");
for (error, expected_code, expected_message) in [
(
ApiError::Retryable {
message: "retry later".to_string(),
- delay: Some(retry_delay),
+ retry_after: Some(retry_after),
},
CodexErrorInfo::Other,
"stream disconnected before completion: retry later",
@@ -25,7 +27,7 @@ fn map_api_error_preserves_retry_delay() {
(
ApiError::RateLimitExceeded {
message: "retry later".to_string(),
- delay: Some(retry_delay),
+ retry_after: Some(retry_after),
},
CodexErrorInfo::RateLimitExceeded,
"rate limit exceeded: retry later",
@@ -36,6 +38,7 @@ fn map_api_error_preserves_retry_delay() {
(
err.to_codex_protocol_error(),
err.retry_delay(/*retry_count*/ 1),
+ err.retry_after(),
err.server_retry_delay(),
err.http_status_code_value(),
err.to_string(),
@@ -43,6 +46,7 @@ fn map_api_error_preserves_retry_delay() {
(
expected_code,
Some(retry_delay),
+ Some(retry_after),
Some(retry_delay),
None,
expected_message.to_string(),
@@ -63,6 +67,7 @@ fn map_api_error_distinguishes_capacity_from_slow_down() {
("unknown_error", CodexErrorInfo::Other, true),
] {
let err = map_api_error(ApiError::Transport(TransportError::Http {
+ retry_after: None,
status: http::StatusCode::SERVICE_UNAVAILABLE,
url: None,
headers: None,
@@ -85,6 +90,7 @@ fn map_api_error_maps_cloudflare_blocked_response_to_user_message() {
let mut headers = HeaderMap::new();
headers.insert(CF_RAY_HEADER, http::HeaderValue::from_static("ray-id"));
let err = map_api_error(ApiError::Transport(TransportError::Http {
+ retry_after: None,
status: http::StatusCode::FORBIDDEN,
url: Some("http://example.com/blocked".to_string()),
headers: Some(headers),
@@ -120,6 +126,7 @@ fn map_api_error_maps_cyber_policy_from_400_body() {
})
.to_string();
let err = map_api_error(ApiError::Transport(TransportError::Http {
+ retry_after: None,
status: http::StatusCode::BAD_REQUEST,
url: Some("http://example.com/v1/responses".to_string()),
headers: None,
@@ -148,6 +155,7 @@ fn map_api_error_maps_wrapped_websocket_cyber_policy_from_400_body() {
})
.to_string();
let err = map_api_error(ApiError::Transport(TransportError::Http {
+ retry_after: None,
status: http::StatusCode::BAD_REQUEST,
url: Some("ws://example.com/v1/responses".to_string()),
headers: None,
@@ -169,6 +177,7 @@ fn map_api_error_uses_cyber_policy_fallback_for_missing_message() {
})
.to_string();
let err = map_api_error(ApiError::Transport(TransportError::Http {
+ retry_after: None,
status: http::StatusCode::BAD_REQUEST,
url: Some("http://example.com/v1/responses".to_string()),
headers: None,
@@ -241,6 +250,7 @@ fn map_api_error_maps_http_and_wrapped_websocket_typed_errors() {
body["status"] = serde_json::json!(400);
}
let err = map_api_error(ApiError::Transport(TransportError::Http {
+ retry_after: None,
status: http::StatusCode::BAD_REQUEST,
url: None,
headers: None,
@@ -275,6 +285,7 @@ fn assert_misalignment_policy_violation_from_http_body(status: http::StatusCode)
})
.to_string();
let err = map_api_error(ApiError::Transport(TransportError::Http {
+ retry_after: None,
status,
url: Some("http://example.com/v1/responses".to_string()),
headers: None,
@@ -308,6 +319,7 @@ fn map_api_error_preserves_misalignment_details_from_403_body() {
})
.to_string();
let err = map_api_error(ApiError::Transport(TransportError::Http {
+ retry_after: None,
status: http::StatusCode::FORBIDDEN,
url: Some("http://example.com/v1/responses".to_string()),
headers: None,
@@ -352,6 +364,7 @@ fn map_api_error_preserves_misalignment_details_from_wrapped_websocket_error() {
})
.to_string();
let err = map_api_error(ApiError::Transport(TransportError::Http {
+ retry_after: None,
status: http::StatusCode::FORBIDDEN,
url: Some("ws://example.com/v1/responses".to_string()),
headers: None,
@@ -393,6 +406,7 @@ fn map_api_error_keeps_other_400_errors_generic() {
})
.to_string();
let err = map_api_error(ApiError::Transport(TransportError::Http {
+ retry_after: None,
status: http::StatusCode::BAD_REQUEST,
url: Some("http://example.com/v1/responses".to_string()),
headers: None,
@@ -426,6 +440,7 @@ fn map_api_error_distinguishes_http_quota_errors_from_rate_limits() {
CodexErrorInfo::UsageLimitExceeded
};
let err = map_api_error(ApiError::Transport(TransportError::Http {
+ retry_after: None,
status: http::StatusCode::TOO_MANY_REQUESTS,
url: None,
headers: None,
@@ -455,6 +470,7 @@ fn map_api_error_maps_usage_limit_limit_name_header() {
})
.to_string();
let err = map_api_error(ApiError::Transport(TransportError::Http {
+ retry_after: None,
status: http::StatusCode::TOO_MANY_REQUESTS,
url: Some("http://example.com/v1/responses".to_string()),
headers: Some(headers),
@@ -488,6 +504,7 @@ fn map_api_error_does_not_fallback_limit_name_to_limit_id() {
})
.to_string();
let err = map_api_error(ApiError::Transport(TransportError::Http {
+ retry_after: None,
status: http::StatusCode::TOO_MANY_REQUESTS,
url: Some("http://example.com/v1/responses".to_string()),
headers: Some(headers),
@@ -537,6 +554,7 @@ fn map_api_error_copies_rate_limit_reached_type_to_usage_limit_snapshot() {
.to_string();
let err = map_api_error(ApiError::Transport(TransportError::Http {
+ retry_after: None,
status: http::StatusCode::TOO_MANY_REQUESTS,
url: Some("http://example.com/v1/responses".to_string()),
headers: Some(headers),
@@ -588,6 +606,7 @@ fn map_api_error_ignores_unparseable_rate_limit_reached_type_headers() {
})
.to_string();
let err = map_api_error(ApiError::Transport(TransportError::Http {
+ retry_after: None,
status: http::StatusCode::TOO_MANY_REQUESTS,
url: Some("http://example.com/v1/responses".to_string()),
headers: Some(headers),
@@ -618,6 +637,7 @@ fn map_api_error_extracts_identity_auth_details_from_headers() {
);
let err = map_api_error(ApiError::Transport(TransportError::Http {
+ retry_after: None,
status: http::StatusCode::UNAUTHORIZED,
url: Some("https://chatgpt.com/backend-api/codex/models".to_string()),
headers: Some(headers),
diff --git a/codex-rs/codex-api/src/endpoint/responses_websocket.rs b/codex-rs/codex-api/src/endpoint/responses_websocket.rs
index 4fe0071ac9..4890ef5cbf 100644
--- a/codex-rs/codex-api/src/endpoint/responses_websocket.rs
+++ b/codex-rs/codex-api/src/endpoint/responses_websocket.rs
@@ -575,6 +575,7 @@ fn map_ws_error(err: WsError, url: &Url) -> ApiError {
url: Some(url.to_string()),
headers: Some(headers),
body,
+ retry_after: None,
})
}
WsError::ConnectionClosed | WsError::AlreadyClosed => {
@@ -637,7 +638,7 @@ fn map_wrapped_websocket_error_event(
.message
.clone()
.unwrap_or_else(|| fallback_message.to_string()),
- delay: None,
+ retry_after: None,
});
}
@@ -651,6 +652,7 @@ fn map_wrapped_websocket_error_event(
url: None,
headers: headers.as_ref().map(json_headers_to_http_headers),
body: Some(original_payload),
+ retry_after: None,
}))
}
@@ -1107,11 +1109,15 @@ mod tests {
.expect("expected websocket error payload to be parsed");
let api_error = map_wrapped_websocket_error_event(wrapped_error, payload)
.expect("expected websocket error payload to map to ApiError");
- let ApiError::Retryable { message, delay } = api_error else {
+ let ApiError::Retryable {
+ message,
+ retry_after,
+ } = api_error
+ else {
panic!("expected ApiError::Retryable");
};
assert_eq!(message, WEBSOCKET_CONNECTION_LIMIT_REACHED_MESSAGE);
- assert_eq!(delay, None);
+ assert_eq!(retry_after, None);
}
#[test]
diff --git a/codex-rs/codex-api/src/error.rs b/codex-rs/codex-api/src/error.rs
index a353a59b8b..b4163b795a 100644
--- a/codex-rs/codex-api/src/error.rs
+++ b/codex-rs/codex-api/src/error.rs
@@ -1,8 +1,8 @@
use crate::rate_limits::RateLimitError;
use codex_client::TransportError;
+use codex_http_client::RetryAfter;
use codex_protocol::protocol::MisalignmentErrorDetails;
use http::StatusCode;
-use std::time::Duration;
use thiserror::Error;
#[derive(Debug, Error)]
@@ -22,12 +22,12 @@ pub enum ApiError {
#[error("retryable error: {message}")]
Retryable {
message: String,
- delay: Option<Duration>,
+ retry_after: Option<RetryAfter>,
},
#[error("rate limit exceeded: {message}")]
RateLimitExceeded {
message: String,
- delay: Option<Duration>,
+ retry_after: Option<RetryAfter>,
},
#[error("rate limit: {0}")]
RateLimit(String),
@@ -45,7 +45,7 @@ pub enum ApiError {
misalignment: Option<MisalignmentErrorDetails>,
},
#[error("server overloaded")]
- ServerOverloaded,
+ ServerOverloaded { retry_after: Option<RetryAfter> },
}
impl From<RateLimitError> for ApiError {
diff --git a/codex-rs/codex-api/src/sse/responses.rs b/codex-rs/codex-api/src/sse/responses.rs
index 9b55b00848..1fb8835318 100644
--- a/codex-rs/codex-api/src/sse/responses.rs
+++ b/codex-rs/codex-api/src/sse/responses.rs
@@ -8,6 +8,7 @@ use crate::safety_buffering::treatment_from_headers;
use crate::telemetry::SseTelemetry;
use codex_client::ByteStream;
use codex_client::StreamResponse;
+use codex_http_client::RetryAfter;
use codex_protocol::ResponseUsageMetadata;
use codex_protocol::models::ResponseItem;
use codex_protocol::protocol::MisalignmentErrorDetails;
@@ -457,15 +458,22 @@ pub fn process_responses_event(
.unwrap_or_else(|| "Invalid request.".to_string());
response_error = ApiError::InvalidPrompt { message };
} else if is_server_overloaded_error(&error) {
- response_error = ApiError::ServerOverloaded;
+ response_error = ApiError::ServerOverloaded { retry_after: None };
} else {
- let delay = try_parse_retry_after(&error);
+ let retry_after =
+ try_parse_retry_delay(&error).and_then(RetryAfter::from_delay);
let message = error.message.unwrap_or_default();
response_error = match error.code.as_deref() {
Some("rate_limit_exceeded" | "slow_down") => {
- ApiError::RateLimitExceeded { message, delay }
+ ApiError::RateLimitExceeded {
+ message,
+ retry_after,
+ }
}
- _ => ApiError::Retryable { message, delay },
+ _ => ApiError::Retryable {
+ message,
+ retry_after,
+ },
};
}
}
@@ -695,7 +703,7 @@ async fn process_sse_with_treatment(
}
}
-fn try_parse_retry_after(err: &Error) -> Option<Duration> {
+fn try_parse_retry_delay(err: &Error) -> Option<Duration> {
if !matches!(
err.code.as_deref(),
Some("rate_limit_exceeded" | "slow_down")
@@ -1115,7 +1123,7 @@ mod tests {
}
}
- #[tokio::test]
+ #[tokio::test(start_paused = true)]
async fn rate_limit_error_preserves_retry_delay() {
let raw_error = r#"{"type":"response.failed","sequence_number":3,"response":{"id":"resp_689bcf18d7f08194bf3440ba62fe05d803fee0cdac429894","object":"response","created_at":1755041560,"status":"failed","background":false,"error":{"code":"rate_limit_exceeded","message":"Rate limit reached for gpt-5.1 in organization org-AAA on tokens per min (TPM): Limit 30000, Used 22999, Requested 12528. Please try again in 11.054s. Visit https://platform.openai.com/account/rate-limits to learn more."}, "usage":null,"user":null,"metadata":{}}}"#;
@@ -1126,12 +1134,18 @@ mod tests {
assert_eq!(events.len(), 1);
match &events[0] {
- Err(ApiError::RateLimitExceeded { message, delay }) => {
+ Err(ApiError::RateLimitExceeded {
+ message,
+ retry_after,
+ }) => {
assert_eq!(
message,
"Rate limit reached for gpt-5.1 in organization org-AAA on tokens per min (TPM): Limit 30000, Used 22999, Requested 12528. Please try again in 11.054s. Visit https://platform.openai.com/account/rate-limits to learn more."
);
- assert_eq!(*delay, Some(Duration::from_secs_f64(11.054)));
+ assert_eq!(
+ retry_after.map(RetryAfter::remaining_delay),
+ Some(Duration::from_secs_f64(11.054))
+ );
}
other => panic!("unexpected rate-limit event: {other:?}"),
}
@@ -1159,7 +1173,7 @@ mod tests {
[
Err(ApiError::RateLimitExceeded {
message: actual,
- delay,
+ retry_after,
}),
],
)
@@ -1168,11 +1182,11 @@ mod tests {
[
Err(ApiError::Retryable {
message: actual,
- delay,
+ retry_after,
}),
],
) => {
- assert_eq!((actual.as_str(), *delay), (message, None));
+ assert_eq!((actual.as_str(), *retry_after), (message, None));
}
_ => panic!("unexpected events for {code}: {events:?}"),
}
@@ -2078,7 +2092,7 @@ mod tests {
}
#[test]
- fn test_try_parse_retry_after() {
+ fn test_try_parse_retry_delay() {
let err = Error {
r#type: None,
message: Some("Rate limit reached for gpt-5.1 in organization org- on tokens per min (TPM): Limit 1, Used 1, Requested 19304. Please try again in 28ms. Visit https://platform.openai.com/account/rate-limits to learn more.".to_string()),
@@ -2088,12 +2102,12 @@ mod tests {
misalignment: None,
};
- let delay = try_parse_retry_after(&err);
+ let delay = try_parse_retry_delay(&err);
assert_eq!(delay, Some(Duration::from_millis(28)));
}
#[test]
- fn test_try_parse_retry_after_no_delay() {
+ fn test_try_parse_retry_delay_no_delay() {
let err = Error {
r#type: None,
message: Some("Rate limit reached for gpt-5.1 in organization <ORG> on tokens per min (TPM): Limit 30000, Used 6899, Requested 24050. Please try again in 1.898s. Visit https://platform.openai.com/account/rate-limits to learn more.".to_string()),
@@ -2102,12 +2116,12 @@ mod tests {
resets_at: None,
misalignment: None,
};
- let delay = try_parse_retry_after(&err);
+ let delay = try_parse_retry_delay(&err);
assert_eq!(delay, Some(Duration::from_secs_f64(1.898)));
}
#[test]
- fn test_try_parse_retry_after_azure() {
+ fn test_try_parse_retry_delay_azure() {
let err = Error {
r#type: None,
message: Some("Rate limit exceeded. Try again in 35 seconds.".to_string()),
@@ -2116,7 +2130,7 @@ mod tests {
resets_at: None,
misalignment: None,
};
- let delay = try_parse_retry_after(&err);
+ let delay = try_parse_retry_delay(&err);
assert_eq!(delay, Some(Duration::from_secs(35)));
}
diff --git a/codex-rs/codex-client/src/retry.rs b/codex-rs/codex-client/src/retry.rs
index 071658272c..b10fa50794 100644
--- a/codex-rs/codex-client/src/retry.rs
+++ b/codex-rs/codex-client/src/retry.rs
@@ -1,4 +1,5 @@
use codex_http_client::Request;
+use codex_http_client::RetryAfter;
use codex_http_client::TransportError;
use rand::Rng;
use std::future::Future;
@@ -99,10 +100,16 @@ where
.should_retry(&err, attempt, policy.max_attempts) =>
{
let retry_attempt = attempt + 1;
- // TODO(anp): Respect Retry-After from HTTP responses before retrying the request.
- let delay = backoff(policy.base_delay, retry_attempt);
+ let retry_after = err.retry_after();
+ let delay = retry_after
+ .map(RetryAfter::remaining_delay)
+ .unwrap_or_else(|| backoff(policy.base_delay, retry_attempt));
crate::record_retry!(retry_attempt, delay, RetryOperation::HttpRequest);
- tokio::time::sleep(delay).await;
+ if let Some(retry_after) = retry_after {
+ tokio::time::sleep_until(retry_after.deadline()).await;
+ } else {
+ tokio::time::sleep(delay).await;
+ }
}
Err(err) => return Err(err),
}
diff --git a/codex-rs/core/src/client_tests.rs b/codex-rs/core/src/client_tests.rs
index 0f62891632..8380e365d7 100644
--- a/codex-rs/core/src/client_tests.rs
+++ b/codex-rs/core/src/client_tests.rs
@@ -214,6 +214,7 @@ async fn workspace_routed_http_rejects_redirects_without_a_routing_header() {
matches!(
result,
Err(TransportError::Http {
+ retry_after: None,
status: http::StatusCode::TEMPORARY_REDIRECT,
..
})
@@ -1411,6 +1412,7 @@ async fn bedrock_unauthorized_error_uses_provider_mapping() {
let url = "https://bedrock-mantle.us-east-2.api.aws/openai/v1/responses";
let error = super::handle_unauthorized(
TransportError::Http {
+ retry_after: None,
status: http::StatusCode::UNAUTHORIZED,
url: Some(url.to_string()),
headers: None,
@@ -1504,6 +1506,7 @@ async fn provider_owned_auth_recovery_is_bounded_and_preserves_unauthorized_fail
assert!(provider.auth_manager().is_none());
let unauthorized = || TransportError::Http {
+ retry_after: None,
status: http::StatusCode::UNAUTHORIZED,
url: Some("https://example.com/v1/responses".to_string()),
headers: None,
diff --git a/codex-rs/core/src/responses_retry.rs b/codex-rs/core/src/responses_retry.rs
index 03acba415c..6094395754 100644
--- a/codex-rs/core/src/responses_retry.rs
+++ b/codex-rs/core/src/responses_retry.rs
@@ -7,10 +7,12 @@ use crate::session::session::Session;
use crate::session::turn_context::TurnContext;
use codex_client::RetryOperation;
use codex_features::Feature;
+use codex_http_client::RetryAfter;
use codex_protocol::error::CodexErr;
use codex_protocol::error::CodexErrorDetails;
use codex_protocol::protocol::EventMsg;
use codex_protocol::protocol::WarningEvent;
+use tokio::time::Instant;
use tracing::warn;
const INITIAL_CONNECTION_RETRY_DELAY: Duration = Duration::from_secs(5);
@@ -64,6 +66,7 @@ pub(crate) async fn handle_response_stream_error(
let Some(delay) = err.retry_delay(retry_count) else {
return Err(err);
};
+ let retry_after = err.retry_after();
if turn_context
.config
@@ -129,8 +132,12 @@ pub(crate) async fn handle_response_stream_error(
)
.await;
}
+ let retry_at = retry_after
+ .map(RetryAfter::deadline)
+ .unwrap_or_else(|| Instant::now() + delay);
+ let delay = retry_at.saturating_duration_since(Instant::now());
codex_client::record_retry!(retry_count, delay, operation);
- tokio::time::sleep(delay).await;
+ tokio::time::sleep_until(retry_at).await;
return Ok(());
}
@@ -138,9 +145,7 @@ pub(crate) async fn handle_response_stream_error(
.thread_extension_data
.insert(ExhaustedResponseRetry {
turn_id: turn_context.sub_id.clone(),
- retry_at: err
- .server_retry_delay()
- .and_then(|delay| tokio::time::Instant::now().checked_add(delay)),
+ retry_at: retry_after.map(RetryAfter::deadline),
});
Err(err)
}
diff --git a/codex-rs/core/src/responses_retry_tests.rs b/codex-rs/core/src/responses_retry_tests.rs
index b6465ded37..85d40eb736 100644
--- a/codex-rs/core/src/responses_retry_tests.rs
+++ b/codex-rs/core/src/responses_retry_tests.rs
@@ -1,8 +1,14 @@
use super::ResponsesStreamRequest;
+use super::ResponsesStreamRetryState;
+use super::handle_response_stream_error;
use super::log_retry;
+use crate::realtime_history::RealtimeHistoryState;
use crate::session::tests::make_session_and_context;
+use codex_http_client::RetryAfter;
use codex_protocol::error::CodexErr;
use std::time::Duration;
+use tokio::sync::Mutex;
+use tokio::time::Instant;
use tracing_test::internal::MockWriter;
#[tokio::test]
@@ -41,3 +47,51 @@ async fn sampling_retry_logs_stream_error_context() {
"sampling_error=stream disconnected before completion: websocket closed by server before response.completed"
));
}
+
+/// Time spent reporting a retry must count toward the server's original deadline.
+#[tokio::test]
+#[expect(
+ clippy::await_holding_invalid_type,
+ reason = "test holds the event-delivery lock to delay notification while virtual time advances"
+)]
+async fn stream_retry_preserves_deadline_across_delayed_notification() {
+ let (mut session, turn_context) = make_session_and_context().await;
+ session.realtime_history = Some(Mutex::new(RealtimeHistoryState::default()));
+ let mut client_session = session.services.model_client.new_session();
+ // The second retry emits a notification even when release builds hide the first.
+ let mut retry_state = ResponsesStreamRetryState {
+ retries: 1,
+ ..Default::default()
+ };
+
+ tokio::time::pause();
+ let advice = RetryAfter::from_delay(Duration::from_secs(10)).expect("retry deadline");
+ tokio::time::advance(Duration::from_secs(4)).await;
+
+ // Hold the event-delivery lock so reporting the error consumes part of the deadline.
+ let history_guard = session.realtime_history.as_ref().unwrap().lock().await;
+ let retry = handle_response_stream_error(
+ &mut retry_state,
+ /*max_retries*/ 2,
+ CodexErr::InternalServerError.with_retry_after(advice),
+ &mut client_session,
+ &session,
+ &turn_context,
+ ResponsesStreamRequest::Sampling,
+ );
+ tokio::pin!(retry);
+ assert!(futures::poll!(&mut retry).is_pending());
+ tokio::time::advance(Duration::from_secs(4)).await;
+ drop(history_guard);
+
+ assert!(futures::poll!(&mut retry).is_pending());
+ tokio::time::advance(Duration::from_secs(1)).await;
+ assert!(futures::poll!(&mut retry).is_pending());
+ retry.await.expect("retry should be allowed");
+ // Tokio rounds timer deadlines up to the next millisecond.
+ let resumed_at = Instant::now();
+ assert!(
+ (advice.deadline()..=advice.deadline() + Duration::from_millis(1)).contains(&resumed_at),
+ "retry resumed at {resumed_at:?}, expected {advice:?}"
+ );
+}
diff --git a/codex-rs/core/tests/suite/retry_after.rs b/codex-rs/core/tests/suite/retry_after.rs
index 4c9e987e9d..76b79f841e 100644
--- a/codex-rs/core/tests/suite/retry_after.rs
+++ b/codex-rs/core/tests/suite/retry_after.rs
@@ -1,8 +1,11 @@
use anyhow::Result;
+use codex_api::ApiError;
+use codex_api::map_api_error;
use codex_client::RetryOn;
use codex_client::RetryPolicy;
use codex_client::run_with_retry;
use codex_http_client::Request;
+use codex_http_client::RetryAfter;
use codex_http_client::TransportError;
use codex_login::CodexAuth;
use codex_models_manager::bundled_models_response;
@@ -272,10 +275,9 @@ async fn wait_for_turn_completion(test: &TestCodex) {
assert_eq!(completed.error, None, "turn should complete successfully");
}
-// TODO(anp) respect Retry-After
-/// HTTP overloads currently retry with local backoff instead of the upstream header delay.
+/// HTTP overloads use the upstream header before another HTTP request.
#[tokio::test(flavor = "current_thread")]
-async fn responses_http_uses_local_backoff_despite_retry_after() -> Result<()> {
+async fn responses_http_uses_retry_after() -> Result<()> {
skip_if_no_network!(Ok(()));
let mut telemetry = RetryTelemetryCapture::install();
@@ -303,7 +305,7 @@ async fn responses_http_uses_local_backoff_despite_retry_after() -> Result<()> {
submit_user_input(&test, "retry the upstream overload").await?;
let retry = telemetry.next_retry().await;
- assert!((FIRST_RETRY_MIN_DELAY..FIRST_RETRY_MAX_DELAY).contains(&retry.delay));
+ assert!(retry.delay <= Duration::from_secs(1));
assert_eq!(
retry,
RetryTelemetryEvent {
@@ -313,7 +315,7 @@ async fn responses_http_uses_local_backoff_despite_retry_after() -> Result<()> {
operation: "request".into(),
}
);
- wait_for_retry(&mut telemetry, &retry).await;
+ assert!(wait_for_retry(&mut telemetry, &retry).await >= Duration::from_secs(1));
wait_for_turn_completion(&test).await;
assert_eq!(response_mock.requests().len(), 2);
@@ -345,6 +347,7 @@ async fn http_retry_backoff_exhausts_attempts() {
.expect("retry attempts should not be poisoned")
.push((attempt, tokio::time::Instant::now()));
std::future::ready(Err::<(), _>(TransportError::Http {
+ retry_after: None,
status: StatusCode::SERVICE_UNAVAILABLE,
url: None,
headers: None,
@@ -376,6 +379,52 @@ async fn http_retry_backoff_exhausts_attempts() {
);
}
+/// Exhausting HTTP retries and mapping the error preserve the last server deadline.
+#[tokio::test(start_paused = true)]
+async fn exhausted_http_retries_preserve_deadline_through_error_mapping() {
+ use tokio::time::Instant;
+
+ let started = Instant::now();
+ let transport_error = run_with_retry(
+ RetryPolicy {
+ max_attempts: 1,
+ base_delay: Duration::from_millis(200),
+ retry_on: RetryOn {
+ retry_429: false,
+ retry_5xx: true,
+ retry_transport: false,
+ },
+ },
+ || Request::new(Method::POST, "http://localhost/v1/responses".into()),
+ |_, attempt| {
+ let elapsed = Instant::now() - started;
+ assert_eq!(elapsed.as_secs(), attempt * 3);
+ let seconds = if attempt == 0 { 3 } else { 10 };
+ std::future::ready(Err::<(), _>(TransportError::Http {
+ status: StatusCode::INTERNAL_SERVER_ERROR,
+ url: None,
+ headers: None,
+ body: None,
+ retry_after: RetryAfter::from_delay(Duration::from_secs(seconds)),
+ }))
+ },
+ )
+ .await
+ .unwrap_err();
+ assert_eq!(Instant::now() - started, Duration::from_secs(3));
+ tokio::time::advance(Duration::from_secs(4)).await;
+ let outer_error = map_api_error(ApiError::Transport(transport_error));
+ let retry_after = outer_error.retry_after().expect("retry advice");
+ assert_eq!(retry_after.deadline(), started + Duration::from_secs(13));
+ assert_eq!(
+ outer_error.server_retry_delay(),
+ Some(Duration::from_secs(6))
+ );
+ tokio::time::advance(Duration::from_secs(6)).await;
+ assert_eq!(outer_error.retry_after(), Some(retry_after));
+ assert_eq!(outer_error.server_retry_delay(), Some(Duration::ZERO));
+}
+
/// Headerless HTTP overloads currently exhaust request retries before emitting one terminal error.
#[tokio::test(flavor = "current_thread")]
async fn responses_http_overload_without_retry_after_exhausts_request_retries() -> Result<()> {
@@ -447,10 +496,9 @@ async fn responses_http_overload_without_retry_after_exhausts_request_retries()
Ok(())
}
-// TODO(anp) respect Retry-After
-/// Remote compaction v2 currently retries with local backoff instead of the upstream header delay.
+/// Remote compaction v2 uses the upstream header before another HTTP request.
#[tokio::test(flavor = "current_thread")]
-async fn compact_v2_uses_local_backoff_despite_retry_after() -> Result<()> {
+async fn compact_v2_uses_retry_after() -> Result<()> {
skip_if_no_network!(Ok(()));
let mut telemetry = RetryTelemetryCapture::install();
@@ -490,7 +538,7 @@ async fn compact_v2_uses_local_backoff_despite_retry_after() -> Result<()> {
test.codex.submit(Op::Compact).await?;
let retry = telemetry.next_retry().await;
- assert!((FIRST_RETRY_MIN_DELAY..FIRST_RETRY_MAX_DELAY).contains(&retry.delay));
+ assert!(retry.delay <= Duration::from_secs(1));
assert_eq!(
retry,
RetryTelemetryEvent {
@@ -500,7 +548,7 @@ async fn compact_v2_uses_local_backoff_despite_retry_after() -> Result<()> {
operation: "request".into(),
}
);
- wait_for_retry(&mut telemetry, &retry).await;
+ assert!(wait_for_retry(&mut telemetry, &retry).await >= Duration::from_secs(1));
wait_for_turn_completion(&test).await;
let requests = response_mock.requests();
@@ -735,11 +783,12 @@ async fn compact_v2_rate_limit_message_uses_server_advised_retry_delay() -> Resu
test.codex.submit(Op::Compact).await?;
let retry = telemetry.next_retry().await;
+ assert!(retry.delay <= Duration::from_secs(1));
assert_eq!(
retry,
RetryTelemetryEvent {
attempt: 1,
- delay: Duration::from_secs(1),
+ delay: retry.delay,
layer: "stream".into(),
operation: "remote_compaction_v2".into(),
}
@@ -809,11 +858,12 @@ async fn compact_v2_rate_limit_message_without_retry_after_uses_server_advised_d
test.codex.submit(Op::Compact).await?;
let retry = telemetry.next_retry().await;
+ assert!(retry.delay <= Duration::from_secs(1));
assert_eq!(
retry,
RetryTelemetryEvent {
attempt: 1,
- delay: Duration::from_secs(1),
+ delay: retry.delay,
layer: "stream".into(),
operation: "remote_compaction_v2".into(),
}
@@ -1134,11 +1184,12 @@ async fn sse_rate_limit_message_uses_server_advised_retry_delay(code: &str) -> R
submit_user_input(&test, "retry after the rate-limit message delay").await?;
let retry = telemetry.next_retry().await;
+ assert!(retry.delay <= Duration::from_secs(1));
assert_eq!(
retry,
RetryTelemetryEvent {
attempt: 1,
- delay: Duration::from_secs(1),
+ delay: retry.delay,
layer: "stream".into(),
operation: "sampling".into(),
}
@@ -1189,11 +1240,12 @@ async fn sse_rate_limit_message_with_retry_after_uses_server_advised_retry_delay
submit_user_input(&test, "retry after both rate-limit delay signals").await?;
let retry = telemetry.next_retry().await;
+ assert!(retry.delay <= Duration::from_secs(1));
assert_eq!(
retry,
RetryTelemetryEvent {
attempt: 1,
- delay: Duration::from_secs(1),
+ delay: retry.delay,
layer: "stream".into(),
operation: "sampling".into(),
}
diff --git a/codex-rs/ext/guardian-v2/src/async_scorer/metrics.rs b/codex-rs/ext/guardian-v2/src/async_scorer/metrics.rs
index 1411c4c5d9..574270492f 100644
--- a/codex-rs/ext/guardian-v2/src/async_scorer/metrics.rs
+++ b/codex-rs/ext/guardian-v2/src/async_scorer/metrics.rs
@@ -53,7 +53,7 @@ pub(super) fn sampler_failure_reason(error: &LunaSamplerError) -> &'static str {
ApiError::CyberPolicy { .. }
| ApiError::BioPolicy { .. }
| ApiError::MisalignmentPolicyViolation { .. } => "policy_error",
- ApiError::ServerOverloaded => "server_overloaded",
+ ApiError::ServerOverloaded { .. } => "server_overloaded",
},
}
}
diff --git a/codex-rs/ext/guardian-v2/src/async_scorer/sampler/execution.rs b/codex-rs/ext/guardian-v2/src/async_scorer/sampler/execution.rs
index 2cd5cd7b94..3a05f0c271 100644
--- a/codex-rs/ext/guardian-v2/src/async_scorer/sampler/execution.rs
+++ b/codex-rs/ext/guardian-v2/src/async_scorer/sampler/execution.rs
@@ -54,7 +54,7 @@ impl SamplingExecution {
ApiError::Retryable { .. }
| ApiError::RateLimitExceeded { .. }
| ApiError::Stream(_)
- | ApiError::ServerOverloaded,
+ | ApiError::ServerOverloaded { .. },
)
| LunaSamplerError::Api(ApiError::Transport(
TransportError::RetryLimit
diff --git a/codex-rs/http-client/Cargo.toml b/codex-rs/http-client/Cargo.toml
index 857798c47c..6bdc70f8b9 100644
--- a/codex-rs/http-client/Cargo.toml
+++ b/codex-rs/http-client/Cargo.toml
@@ -10,6 +10,7 @@ codex-utils-rustls-provider = { workspace = true }
futures = { workspace = true }
http = { workspace = true }
http-body = "1.0.1"
+httpdate = { workspace = true }
native-tls = "0.2"
opentelemetry = { workspace = true }
reqwest = { workspace = true, features = ["json", "rustls-tls-native-roots", "stream"] }
@@ -45,6 +46,7 @@ opentelemetry_sdk = { workspace = true }
pretty_assertions = { workspace = true }
rcgen = { workspace = true }
tempfile = { workspace = true }
+tokio = { workspace = true, features = ["test-util"] }
tracing-subscriber = { workspace = true }
[lib]
diff --git a/codex-rs/http-client/src/error.rs b/codex-rs/http-client/src/error.rs
index 6105b2eea8..181a213b14 100644
--- a/codex-rs/http-client/src/error.rs
+++ b/codex-rs/http-client/src/error.rs
@@ -1,6 +1,7 @@
//! Errors returned by the shared Codex HTTP transport.
use crate::client::HttpError;
+use crate::retry_after::RetryAfter;
use http::HeaderMap;
use http::StatusCode;
use thiserror::Error;
@@ -15,6 +16,7 @@ pub enum TransportError {
url: Option<String>,
headers: Option<HeaderMap>,
body: Option<String>,
+ retry_after: Option<RetryAfter>,
},
#[error("retry limit reached")]
RetryLimit,
@@ -30,6 +32,22 @@ pub enum TransportError {
ResponseTooLarge { max_bytes: usize },
}
+impl TransportError {
+ /// Returns retry advice captured when this error response arrived.
+ pub fn retry_after(&self) -> Option<RetryAfter> {
+ match self {
+ Self::Http { retry_after, .. } => *retry_after,
+ Self::Policy(_)
+ | Self::RetryLimit
+ | Self::Timeout
+ | Self::Connection(_)
+ | Self::Network(_)
+ | Self::Build(_)
+ | Self::ResponseTooLarge { .. } => None,
+ }
+ }
+}
+
#[derive(Debug, Error)]
pub enum StreamError {
#[error("stream failed: {0}")]
diff --git a/codex-rs/http-client/src/lib.rs b/codex-rs/http-client/src/lib.rs
index 0a2bf7e5b3..5eb01f021b 100644
--- a/codex-rs/http-client/src/lib.rs
+++ b/codex-rs/http-client/src/lib.rs
@@ -11,6 +11,7 @@ mod request;
mod request_builder;
mod request_draft;
mod response;
+mod retry_after;
mod route_aware_client_pool;
mod route_aware_redirect;
mod tls_backend_fallback;
@@ -61,6 +62,7 @@ pub use crate::request::RequestCompression;
pub use crate::request::Response;
pub use crate::request_builder::RequestBuilder;
pub use crate::response::HttpResponse;
+pub use crate::retry_after::RetryAfter;
pub use crate::route_aware_client_pool::RouteAwareClientPool;
pub use crate::route_aware_client_pool::RouteAwareClientPoolError;
pub use crate::route_aware_client_pool::RouteAwareRequestError;
diff --git a/codex-rs/http-client/src/retry_after.rs b/codex-rs/http-client/src/retry_after.rs
new file mode 100644
index 0000000000..9da9e0fb9e
--- /dev/null
+++ b/codex-rs/http-client/src/retry_after.rs
@@ -0,0 +1,54 @@
+//! Server retry advice captured once so passing an error between callers does not restart it.
+
+use http::HeaderMap;
+use http::header::RETRY_AFTER;
+use std::time::Duration;
+use std::time::SystemTime;
+use tokio::time::Instant;
+
+/// The earliest time a server advised making a follow-up request.
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+pub struct RetryAfter(Instant);
+
+impl RetryAfter {
+ /// Captures an interval measured from receipt of the server's advice.
+ pub fn from_delay(delay: Duration) -> Option<Self> {
+ Instant::now().checked_add(delay).map(Self)
+ }
+
+ /// Reads either nonnegative delay seconds or an HTTP date from `Retry-After`.
+ pub fn from_headers(headers: &HeaderMap) -> Option<Self> {
+ Self::from_header(headers.get(RETRY_AFTER)?.to_str().ok()?)
+ }
+
+ /// Reads a `Retry-After` value, including values carried inside a streamed error.
+ pub fn from_header(value: &str) -> Option<Self> {
+ // Sample wall time first so a scheduling pause cannot shorten the server's deadline.
+ let now = SystemTime::now();
+ let received_at = Instant::now();
+ let value = value.trim();
+ let delay = if !value.is_empty() && value.bytes().all(|byte| byte.is_ascii_digit()) {
+ Duration::from_secs(value.parse().ok()?)
+ } else {
+ httpdate::parse_http_date(value)
+ .ok()?
+ .duration_since(now)
+ .unwrap_or_default()
+ };
+ received_at.checked_add(delay).map(Self)
+ }
+
+ /// Returns the original deadline for callers that pass advice on or sleep until it.
+ pub fn deadline(self) -> Instant {
+ self.0
+ }
+
+ /// Returns the remaining delay, or zero after the deadline has passed.
+ pub fn remaining_delay(self) -> Duration {
+ self.0.saturating_duration_since(Instant::now())
+ }
+}
+
+#[cfg(test)]
+#[path = "retry_after_tests.rs"]
+mod tests;
diff --git a/codex-rs/http-client/src/retry_after_tests.rs b/codex-rs/http-client/src/retry_after_tests.rs
new file mode 100644
index 0000000000..3250d7696b
--- /dev/null
+++ b/codex-rs/http-client/src/retry_after_tests.rs
@@ -0,0 +1,45 @@
+use super::*;
+use http::HeaderValue;
+use pretty_assertions::assert_eq;
+
+/// Delta seconds start at receipt and remain present once their deadline expires.
+#[tokio::test(start_paused = true)]
+async fn delta_seconds_count_down_to_zero() {
+ let mut headers = HeaderMap::new();
+ headers.insert(RETRY_AFTER, HeaderValue::from_static("10"));
+ let advice = RetryAfter::from_headers(&headers).expect("valid advice");
+ let deadline = Instant::now() + Duration::from_secs(10);
+ assert_eq!(advice.deadline(), deadline);
+ assert_eq!(advice.remaining_delay(), Duration::from_secs(10));
+ tokio::time::advance(Duration::from_secs(4)).await;
+ assert_eq!(advice.remaining_delay(), Duration::from_secs(6));
+ tokio::time::advance(Duration::from_secs(7)).await;
+ assert_eq!(advice.remaining_delay(), Duration::ZERO);
+}
+
+/// HTTP dates share the monotonic clock; expired dates and explicit zero are still advice.
+#[tokio::test(start_paused = true)]
+async fn http_dates_and_invalid_advice() {
+ let value = httpdate::fmt_http_date(SystemTime::now() + Duration::from_secs(60));
+ let date = httpdate::parse_http_date(&value).expect("formatted date");
+ let before = SystemTime::now();
+ let advice = RetryAfter::from_header(&value).expect("valid date");
+ let after = SystemTime::now();
+ let remaining = advice.remaining_delay();
+ assert!(
+ (date.duration_since(after).unwrap()..=date.duration_since(before).unwrap())
+ .contains(&remaining)
+ );
+ tokio::time::advance(Duration::from_secs(3)).await;
+ assert_eq!(advice.remaining_delay(), remaining - Duration::from_secs(3));
+
+ for value in ["0", "Thu, 01 Jan 1970 00:00:00 GMT"] {
+ assert_eq!(
+ RetryAfter::from_header(value).map(RetryAfter::remaining_delay),
+ Some(Duration::ZERO)
+ );
+ }
+ for value in ["", "-1", "+1", "1.5", "invalid", "18446744073709551615"] {
+ assert_eq!(RetryAfter::from_header(value), None, "{value}");
+ }
+}
diff --git a/codex-rs/http-client/src/transport.rs b/codex-rs/http-client/src/transport.rs
index 9329f5e804..369d4ea68c 100644
--- a/codex-rs/http-client/src/transport.rs
+++ b/codex-rs/http-client/src/transport.rs
@@ -13,6 +13,7 @@ use crate::request::Request;
use crate::request::RequestBody;
use crate::request::Response;
use crate::request_draft::RequestDraft;
+use crate::retry_after::RetryAfter;
use bytes::Bytes;
use bytes::BytesMut;
use futures::StreamExt;
@@ -179,6 +180,7 @@ impl HttpTransport for ReqwestTransport {
let resp = self.send(req).await?;
let status = resp.status();
let headers = resp.headers().clone();
+ let retry_after = RetryAfter::from_headers(&headers);
let bytes = match response_body_limit_bytes {
Some(max_bytes) => bounded_response_bytes(resp, max_bytes).await,
None => resp.bytes().await.map_err(Self::map_error),
@@ -191,6 +193,8 @@ impl HttpTransport for ReqwestTransport {
) => return Err(error),
// Keep bounded diagnostic-body failures from hiding HTTP auth/retry status.
Err(_) if response_body_limit_bytes.is_some() => None,
+ // TODO(anp): Preserve retry advice on body failures without changing their
+ // network/timeout classification to HTTP status handling.
Err(error) => return Err(error),
};
return Err(TransportError::Http {
@@ -198,6 +202,7 @@ impl HttpTransport for ReqwestTransport {
url: Some(url),
headers: Some(headers),
body,
+ retry_after,
});
}
Ok(Response {
@@ -216,6 +221,7 @@ impl HttpTransport for ReqwestTransport {
let status = resp.status();
let headers = resp.headers().clone();
if !status.is_success() {
+ let retry_after = RetryAfter::from_headers(&headers);
let body = match response_body_limit_bytes {
Some(max_bytes) => match bounded_response_bytes(resp, max_bytes).await {
Ok(bytes) => {
@@ -242,6 +248,7 @@ impl HttpTransport for ReqwestTransport {
url: Some(url),
headers: Some(headers),
body,
+ retry_after,
});
}
let bytes = match response_body_limit_bytes {
diff --git a/codex-rs/model-provider/src/amazon_bedrock/error.rs b/codex-rs/model-provider/src/amazon_bedrock/error.rs
index 5274029daa..a69bd6524e 100644
--- a/codex-rs/model-provider/src/amazon_bedrock/error.rs
+++ b/codex-rs/model-provider/src/amazon_bedrock/error.rs
@@ -27,8 +27,8 @@ pub(super) fn map_api_error(error: ApiError) -> CodexErr {
let mut response = response.clone();
response.user_message = Some(BEDROCK_EXPIRED_SIGNATURE_MESSAGE.to_string());
let mapped_error = CodexErr::new(CodexErrorDetails::UnexpectedStatus(response));
- return match error.server_retry_delay() {
- Some(retry_delay) => mapped_error.with_retry_delay(retry_delay),
+ return match error.retry_after() {
+ Some(retry_after) => mapped_error.with_retry_after(retry_after),
None => mapped_error,
};
}
diff --git a/codex-rs/model-provider/src/amazon_bedrock/error_tests.rs b/codex-rs/model-provider/src/amazon_bedrock/error_tests.rs
index e08d61f811..c53f0b71b6 100644
--- a/codex-rs/model-provider/src/amazon_bedrock/error_tests.rs
+++ b/codex-rs/model-provider/src/amazon_bedrock/error_tests.rs
@@ -1,7 +1,9 @@
use std::num::NonZeroU64;
+use std::time::Duration;
use codex_api::ApiError;
use codex_api::TransportError;
+use codex_http_client::RetryAfter;
use codex_model_provider_info::AwsCredentialExportConfig;
use codex_model_provider_info::ModelProviderAwsAuthInfo;
use codex_model_provider_info::ModelProviderInfo;
@@ -24,6 +26,7 @@ fn http_error(status: StatusCode, body: &str) -> ApiError {
let mut headers = HeaderMap::new();
headers.insert("x-request-id", HeaderValue::from_static("req-bedrock"));
ApiError::Transport(TransportError::Http {
+ retry_after: None,
status,
url: Some(BEDROCK_RESPONSES_URL.to_string()),
headers: Some(headers),
@@ -53,6 +56,20 @@ fn expired_signature_has_actionable_guidance() {
);
}
+/// Rewording an expired-signature error must not replace the server's original deadline.
+#[test]
+fn expired_signature_preserves_the_retry_deadline() {
+ let advice = RetryAfter::from_delay(Duration::from_secs(10)).expect("retry advice");
+ let error = map_api_error(ApiError::Transport(TransportError::Http {
+ status: StatusCode::UNAUTHORIZED,
+ url: None,
+ headers: None,
+ body: Some("Signature expired: old is now earlier than new".into()),
+ retry_after: Some(advice),
+ }));
+ assert_eq!(error.retry_after(), Some(advice));
+}
+
#[test]
fn other_unauthorized_errors_remain_generic() {
let error = map_api_error(http_error(
@@ -123,6 +140,7 @@ fn classifies_only_refreshable_bedrock_auth_failures() {
for (status, body, expected) in cases {
let error = TransportError::Http {
+ retry_after: None,
status,
url: Some(BEDROCK_RESPONSES_URL.to_string()),
headers: None,
diff --git a/codex-rs/model-provider/src/provider.rs b/codex-rs/model-provider/src/provider.rs
index 07b34c39e4..1730e2a75b 100644
--- a/codex-rs/model-provider/src/provider.rs
+++ b/codex-rs/model-provider/src/provider.rs
@@ -975,6 +975,7 @@ mod tests {
(http::StatusCode::FORBIDDEN, "AccessDeniedException", false),
] {
let error = TransportError::Http {
+ retry_after: None,
status,
url: None,
headers: None,
diff --git a/codex-rs/protocol/Cargo.toml b/codex-rs/protocol/Cargo.toml
index 4e2bf12aa5..0ca5117c8c 100644
--- a/codex-rs/protocol/Cargo.toml
+++ b/codex-rs/protocol/Cargo.toml
@@ -59,6 +59,7 @@ seccompiler = { workspace = true }
anyhow = { workspace = true }
pretty_assertions = { workspace = true }
tempfile = { workspace = true }
+tokio = { workspace = true, features = ["test-util"] }
[package.metadata.cargo-shear]
# Required because:
diff --git a/codex-rs/protocol/src/error.rs b/codex-rs/protocol/src/error.rs
index 14825e4db6..814cb63fc5 100644
--- a/codex-rs/protocol/src/error.rs
+++ b/codex-rs/protocol/src/error.rs
@@ -18,6 +18,7 @@ use chrono::Utc;
use codex_async_utils::CancelErr;
use codex_async_utils::backoff;
use codex_http_client::HttpError;
+use codex_http_client::RetryAfter;
use codex_utils_string::truncate_middle_chars;
use codex_utils_string::truncate_middle_with_token_budget;
use http::StatusCode;
@@ -71,7 +72,7 @@ pub enum SandboxErr {
pub struct CodexErr {
details: CodexErrorDetails,
- server_retry_delay: Option<Duration>,
+ retry_after: Option<RetryAfter>,
}
/// The semantic category and diagnostic payload for a [`CodexErr`].
@@ -207,7 +208,7 @@ impl fmt::Debug for CodexErr {
CodexErrorDetails::Stream(message) => formatter
.debug_tuple("Stream")
.field(message)
- .field(&self.server_retry_delay)
+ .field(&self.server_retry_delay())
.finish(),
details => fmt::Debug::fmt(details, formatter),
}
@@ -230,7 +231,7 @@ impl From<CodexErrorDetails> for CodexErr {
fn from(details: CodexErrorDetails) -> Self {
Self {
details,
- server_retry_delay: None,
+ retry_after: None,
}
}
}
@@ -294,7 +295,7 @@ macro_rules! codex_err_unit_constructors {
#[allow(non_upper_case_globals)]
pub const $variant: Self = Self {
details: CodexErrorDetails::$variant,
- server_retry_delay: None,
+ retry_after: None,
};
)*
};
@@ -418,7 +419,7 @@ impl CodexErr {
| CodexErrorDetails::Io(_)
| CodexErrorDetails::Json(_)
| CodexErrorDetails::TokioJoin(_) => Some(
- self.server_retry_delay
+ self.server_retry_delay()
.unwrap_or_else(|| backoff(retry_count)),
),
#[cfg(target_os = "linux")]
@@ -426,13 +427,20 @@ impl CodexErr {
}
}
- /// Returns only the delay advised by the server, without applying local retry policy.
+ /// Returns the original server-advised instant for callers that pass the error on.
+ pub fn retry_after(&self) -> Option<RetryAfter> {
+ self.retry_after
+ }
+
+ /// Returns the remaining server-advised delay without applying local retry policy.
+ /// Expired advice stays present as zero instead of falling back to a local delay.
pub fn server_retry_delay(&self) -> Option<Duration> {
- self.server_retry_delay
+ self.retry_after.map(RetryAfter::remaining_delay)
}
- pub fn with_retry_delay(mut self, retry_delay: Duration) -> Self {
- self.server_retry_delay = Some(retry_delay);
+ /// Retains an already captured server deadline without restarting it.
+ pub fn with_retry_after(mut self, retry_after: RetryAfter) -> Self {
+ self.retry_after = Some(retry_after);
self
}
diff --git a/codex-rs/protocol/src/error_tests.rs b/codex-rs/protocol/src/error_tests.rs
index 8819c7fd68..837feef512 100644
--- a/codex-rs/protocol/src/error_tests.rs
+++ b/codex-rs/protocol/src/error_tests.rs
@@ -11,13 +11,17 @@ use http::StatusCode;
use pretty_assertions::assert_eq;
use std::time::Duration;
-#[test]
-fn codex_err_debug_preserves_legacy_shape() {
+#[tokio::test(start_paused = true)]
+async fn codex_err_debug_preserves_legacy_shape() {
let actual = [
CodexErr::Timeout,
CodexErr::Stream("disconnected".to_string()),
- CodexErr::Stream("retry later".to_string()).with_retry_delay(Duration::from_secs(2)),
- CodexErr::InternalServerError.with_retry_delay(Duration::from_secs(3)),
+ CodexErr::Stream("retry later".to_string()).with_retry_after(
+ RetryAfter::from_delay(Duration::from_secs(2)).expect("retry deadline"),
+ ),
+ CodexErr::InternalServerError.with_retry_after(
+ RetryAfter::from_delay(Duration::from_secs(3)).expect("retry deadline"),
+ ),
]
.map(|err| format!("{err:?}"));
@@ -88,7 +92,8 @@ fn retry_delay_distinguishes_server_advice_backoff_and_terminal_errors() {
assert_eq!(error.server_retry_delay(), None);
let advice = Duration::ZERO;
- let error = error.with_retry_delay(advice);
+ let retry_after = RetryAfter::from_delay(advice).expect("retry deadline");
+ let error = error.with_retry_after(retry_after);
assert_eq!(
(
error.retry_delay(/*retry_count*/ 1),
@@ -98,7 +103,7 @@ fn retry_delay_distinguishes_server_advice_backoff_and_terminal_errors() {
(Some(advice), Some(advice), Some(advice)),
);
- let error = CodexErr::QuotaExceeded.with_retry_delay(advice);
+ let error = CodexErr::QuotaExceeded.with_retry_after(retry_after);
assert_eq!(
(
error.retry_delay(/*retry_count*/ 1),
diff --git a/codex-rs/response-debug-context/src/lib.rs b/codex-rs/response-debug-context/src/lib.rs
index ad65c0b1fc..98e2a6892c 100644
--- a/codex-rs/response-debug-context/src/lib.rs
+++ b/codex-rs/response-debug-context/src/lib.rs
@@ -87,7 +87,7 @@ pub fn telemetry_api_error_message(error: &ApiError) -> String {
ApiError::CyberPolicy { .. } => "cyber policy".to_string(),
ApiError::BioPolicy { .. } => "bio policy".to_string(),
ApiError::MisalignmentPolicyViolation { .. } => "misalignment policy violation".to_string(),
- ApiError::ServerOverloaded => "server overloaded".to_string(),
+ ApiError::ServerOverloaded { .. } => "server overloaded".to_string(),
}
}
@@ -119,6 +119,7 @@ mod tests {
);
let context = extract_response_debug_context(&TransportError::Http {
+ retry_after: None,
status: StatusCode::UNAUTHORIZED,
url: Some("https://chatgpt.com/backend-api/codex/models".to_string()),
headers: Some(headers),
@@ -139,6 +140,7 @@ mod tests {
#[test]
fn telemetry_error_messages_omit_upstream_bodies() {
let transport = TransportError::Http {
+ retry_after: None,
status: StatusCode::UNAUTHORIZED,
url: Some("https://chatgpt.com/backend-api/codex/responses".to_string()),
headers: None,
@@ -153,7 +155,7 @@ mod tests {
assert_eq!(
telemetry_api_error_message(&ApiError::RateLimitExceeded {
message: "private upstream diagnostic".to_string(),
- delay: Some(std::time::Duration::from_secs(1)),
+ retry_after: None,
}),
"rate limit exceeded"
);