Lightweight coding agent that runs in your terminal
Rust
Centralize retry decisions and delays in `CodexErr` (#46540)
Adam Perry @ OpenAId5b29951aca5
diff --git a/codex-rs/codex-api/src/api_bridge_tests.rs b/codex-rs/codex-api/src/api_bridge_tests.rsindex 882c1fdf6c..595232d2d9 100644--- a/codex-rs/codex-api/src/api_bridge_tests.rs+++ b/codex-rs/codex-api/src/api_bridge_tests.rs@@ -35,15 +35,15 @@ fn map_api_error_preserves_retry_delay() {assert_eq!((err.to_codex_protocol_error(),- err.retry_delay(),- err.is_retryable(),+ err.retry_delay(/*retry_count*/ 1),+ err.server_retry_delay(),err.http_status_code_value(),err.to_string(),),(expected_code,Some(retry_delay),- true,+ Some(retry_delay),None,expected_message.to_string(),)@@ -71,7 +71,10 @@ fn map_api_error_distinguishes_capacity_from_slow_down() {),}));assert_eq!(- (err.to_codex_protocol_error(), err.is_retryable()),+ (+ err.to_codex_protocol_error(),+ err.retry_delay(/*retry_count*/ 1).is_some()+ ),(expected, retryable));}@@ -188,7 +191,7 @@ fn map_api_error_preserves_bio_policy() {});assert_eq!(err.to_codex_protocol_error(), CodexErrorInfo::BioPolicy);assert_eq!(err.to_string(), "This request was blocked by bio policy.");- assert!(!err.is_retryable());+ assert_eq!(err.retry_delay(/*retry_count*/ 1), None);}#[test]@@ -223,7 +226,7 @@ fn map_api_error_maps_http_and_wrapped_websocket_bio_policy() {};assert_eq!(message, expected);assert_eq!(err.to_codex_protocol_error(), CodexErrorInfo::BioPolicy);- assert!(!err.is_retryable());+ assert_eq!(err.retry_delay(/*retry_count*/ 1), None);}}}@@ -263,7 +266,7 @@ fn assert_misalignment_policy_violation_from_http_body(status: http::StatusCode)};assert_eq!(message, "This request violated the misalignment policy.");assert_eq!(misalignment, &None);- assert!(!err.is_retryable());+ assert_eq!(err.retry_delay(/*retry_count*/ 1), None);}#[test]@@ -305,7 +308,7 @@ fn map_api_error_preserves_misalignment_details_from_403_body() {}),}));- assert!(!err.is_retryable());+ assert_eq!(err.retry_delay(/*retry_count*/ 1), None);}#[test]@@ -352,7 +355,7 @@ fn map_api_error_preserves_misalignment_details_from_wrapped_websocket_error() {}),}));- assert!(!err.is_retryable());+ assert_eq!(err.retry_delay(/*retry_count*/ 1), None);}#[test]diff --git a/codex-rs/codex-client/src/retry.rs b/codex-rs/codex-client/src/retry.rsindex 86071851f6..fd6bf007d8 100644--- a/codex-rs/codex-client/src/retry.rs+++ b/codex-rs/codex-client/src/retry.rs@@ -98,6 +98,7 @@ where.should_retry(&err, attempt, policy.max_attempts) =>{let retry_attempt = attempt + 1;+ // TODO(anp): Respect Retry-After from HTTP responses before retrying the request.let delay = backoff(policy.base_delay, retry_attempt);crate::record_retry!(retry_attempt, delay, RetryOperation::HttpRequest);tokio::time::sleep(delay).await;diff --git a/codex-rs/core/src/client.rs b/codex-rs/core/src/client.rsindex 922e57c4f7..1c5b49353a 100644--- a/codex-rs/core/src/client.rs+++ b/codex-rs/core/src/client.rs@@ -2577,7 +2577,7 @@ async fn handle_unauthorized(original_error = %original,"provider authentication recovery failed");- return Err(if error.is_retryable() {+ return Err(if error.retry_delay(/*retry_count*/ 1).is_some() {original} else {errordiff --git a/codex-rs/core/src/compact_remote_v2.rs b/codex-rs/core/src/compact_remote_v2.rsindex 680deca87c..e0dd97b999 100644--- a/codex-rs/core/src/compact_remote_v2.rs+++ b/codex-rs/core/src/compact_remote_v2.rs@@ -24,7 +24,7 @@ use crate::responses_metadata::CodexResponsesMetadata;use crate::responses_metadata::CompactionTurnMetadata;use crate::responses_retry::ResponsesStreamRequest;use crate::responses_retry::ResponsesStreamRetryState;-use crate::responses_retry::handle_retryable_response_stream_error;+use crate::responses_retry::handle_response_stream_error;use crate::session::RequestEffortUsage;use crate::session::session::Session;use crate::session::step_context::StepContext;@@ -417,9 +417,8 @@ async fn run_remote_compaction_request_v2(match result {Ok(compaction_output) => return Ok(compaction_output),- Err(err) if !err.is_retryable() => return Err(err),Err(err) => {- handle_retryable_response_stream_error(+ handle_response_stream_error(&mut retry_state,max_retries,err,diff --git a/codex-rs/core/src/responses_retry.rs b/codex-rs/core/src/responses_retry.rsindex 462bb95ba8..03acba415c 100644--- a/codex-rs/core/src/responses_retry.rs+++ b/codex-rs/core/src/responses_retry.rs@@ -5,7 +5,6 @@ use std::time::Duration;use crate::client::ModelClientSession;use crate::session::session::Session;use crate::session::turn_context::TurnContext;-use crate::util::backoff;use codex_client::RetryOperation;use codex_features::Feature;use codex_protocol::error::CodexErr;@@ -46,9 +45,9 @@ pub(crate) struct ExhaustedResponseRetry {pub(crate) retry_at: Option<tokio::time::Instant>,}-/// Handles a retryable stream error and returns `Ok(())` when the caller should-/// retry the request loop.-pub(crate) async fn handle_retryable_response_stream_error(+/// Returns `Ok(())` when the caller should retry the request loop, or the original error when+/// it is terminal or the retry budget is exhausted.+pub(crate) async fn handle_response_stream_error(retry_state: &mut ResponsesStreamRetryState,max_retries: u64,err: CodexErr,@@ -61,6 +60,10 @@ pub(crate) async fn handle_retryable_response_stream_error(ResponsesStreamRequest::Sampling => RetryOperation::Sampling,ResponsesStreamRequest::RemoteCompactionV2 => RetryOperation::RemoteCompactionV2,};+ let retry_count = retry_state.retries.saturating_add(1);+ let Some(delay) = err.retry_delay(retry_count) else {+ return Err(err);+ };if turn_context.config@@ -89,6 +92,7 @@ pub(crate) async fn handle_retryable_response_stream_error(return Ok(());}+ // TODO(anp): Respect server retry advice before issuing the fallback HTTP request.if retry_state.retries >= max_retries&& client_session.try_switch_fallback_transport(&turn_context.session_telemetry,@@ -107,9 +111,7 @@ pub(crate) async fn handle_retryable_response_stream_error(}if retry_state.retries < max_retries {- retry_state.retries += 1;- let retry_count = retry_state.retries;- let delay = err.retry_delay().unwrap_or_else(|| backoff(retry_count));+ retry_state.retries = retry_count;log_retry(request, turn_context, &err, retry_count, max_retries, delay);// In release builds, hide the first websocket retry notification to reduce noisy@@ -137,7 +139,7 @@ pub(crate) async fn handle_retryable_response_stream_error(.insert(ExhaustedResponseRetry {turn_id: turn_context.sub_id.clone(),retry_at: err- .retry_delay()+ .server_retry_delay().and_then(|delay| tokio::time::Instant::now().checked_add(delay)),});Err(err)diff --git a/codex-rs/core/src/session/turn.rs b/codex-rs/core/src/session/turn.rsindex 7b624d8509..3f396ef5db 100644--- a/codex-rs/core/src/session/turn.rs+++ b/codex-rs/core/src/session/turn.rs@@ -32,7 +32,7 @@ use crate::responses_metadata::CodexResponsesMetadata;use crate::responses_metadata::CodexResponsesRequestKind;use crate::responses_retry::ResponsesStreamRequest;use crate::responses_retry::ResponsesStreamRetryState;-use crate::responses_retry::handle_retryable_response_stream_error;+use crate::responses_retry::handle_response_stream_error;use crate::session::PreviousTurnSettings;use crate::session::TurnInput;use crate::session::daemon_recovery::RecordedTurnInput;@@ -1644,11 +1644,7 @@ async fn run_sampling_request(original_input = Some(prompt.input);}- if !err.is_retryable() {- return Err(err);- }-- handle_retryable_response_stream_error(+ handle_response_stream_error(&mut retry_state,max_retries,err,diff --git a/codex-rs/model-provider/src/amazon_bedrock/credential_export_tests.rs b/codex-rs/model-provider/src/amazon_bedrock/credential_export_tests.rsindex b027757a61..fa5db51a8e 100644--- a/codex-rs/model-provider/src/amazon_bedrock/credential_export_tests.rs+++ b/codex-rs/model-provider/src/amazon_bedrock/credential_export_tests.rs@@ -324,10 +324,11 @@ async fn credential_export_reuses_cache_and_coalesces_refreshes() -> io::Result<}let (credentials, refresh) = tokio::join!(credentials, refresh);assert_eq!(credentials?, rotated);- assert!(- !refresh+ assert_eq!(+ refresh.expect_err("queued login recovery must still run")- .is_retryable()+ .retry_delay(/*retry_count*/ 1),+ None);assert!(exporter.cached.lock().await.is_none());diff --git a/codex-rs/model-provider/src/amazon_bedrock/error.rs b/codex-rs/model-provider/src/amazon_bedrock/error.rsindex dc71b47979..5274029daa 100644--- a/codex-rs/model-provider/src/amazon_bedrock/error.rs+++ b/codex-rs/model-provider/src/amazon_bedrock/error.rs@@ -27,7 +27,7 @@ pub(super) fn map_api_error(error: ApiError) -> CodexErr {let mut response = response.clone();response.user_message = Some(BEDROCK_EXPIRED_SIGNATURE_MESSAGE.to_string());let mapped_error = CodexErr::new(CodexErrorDetails::UnexpectedStatus(response));- return match error.retry_delay() {+ return match error.server_retry_delay() {Some(retry_delay) => mapped_error.with_retry_delay(retry_delay),None => mapped_error,};diff --git a/codex-rs/model-provider/src/amazon_bedrock/error_tests.rs b/codex-rs/model-provider/src/amazon_bedrock/error_tests.rsindex 7ecfeda5a5..e08d61f811 100644--- a/codex-rs/model-provider/src/amazon_bedrock/error_tests.rs+++ b/codex-rs/model-provider/src/amazon_bedrock/error_tests.rs@@ -208,6 +208,12 @@ fn credential_export_errors_distinguish_configuration_failures_from_retryable_auunreachable!("all test cases are transport errors");};assert_eq!(provider.is_recoverable_auth_error(transport), recoverable);- assert_eq!(provider.map_api_error(error).is_retryable(), retryable);+ assert_eq!(+ provider+ .map_api_error(error)+ .retry_delay(/*retry_count*/ 1)+ .is_some(),+ retryable+ );}}diff --git a/codex-rs/model-provider/src/provider.rs b/codex-rs/model-provider/src/provider.rsindex 691d75bfba..5f7bd4b035 100644--- a/codex-rs/model-provider/src/provider.rs+++ b/codex-rs/model-provider/src/provider.rs@@ -993,7 +993,7 @@ mod tests {.recover_from_unauthorized().await.expect_err("non-aws command should be rejected");- assert!(!error.is_retryable());+ assert_eq!(error.retry_delay(/*retry_count*/ 1), None);assert_eq!(error.to_string(), "AWS auth refresh command must be `aws`");let (first_result, second_result) = tokio::join!(diff --git a/codex-rs/protocol/src/error.rs b/codex-rs/protocol/src/error.rsindex b62943646d..609cdcab95 100644--- a/codex-rs/protocol/src/error.rs+++ b/codex-rs/protocol/src/error.rs@@ -16,6 +16,7 @@ use chrono::Datelike;use chrono::Local;use chrono::Utc;use codex_async_utils::CancelErr;+use codex_async_utils::backoff;use codex_http_client::HttpError;use codex_utils_string::truncate_middle_chars;use codex_utils_string::truncate_middle_with_token_budget;@@ -70,7 +71,7 @@ pub enum SandboxErr {pub struct CodexErr {details: CodexErrorDetails,- retry_delay: Option<Duration>,+ server_retry_delay: Option<Duration>,}/// The semantic category and diagnostic payload for a [`CodexErr`].@@ -204,7 +205,7 @@ impl fmt::Debug for CodexErr {CodexErrorDetails::Stream(message) => formatter.debug_tuple("Stream").field(message)- .field(&self.retry_delay)+ .field(&self.server_retry_delay).finish(),details => fmt::Debug::fmt(details, formatter),}@@ -227,7 +228,7 @@ impl From<CodexErrorDetails> for CodexErr {fn from(details: CodexErrorDetails) -> Self {Self {details,- retry_delay: None,+ server_retry_delay: None,}}}@@ -291,7 +292,7 @@ macro_rules! codex_err_unit_constructors {#[allow(non_upper_case_globals)]pub const $variant: Self = Self {details: CodexErrorDetails::$variant,- retry_delay: None,+ server_retry_delay: None,};)*};@@ -371,7 +372,11 @@ impl CodexErr {&self.details}- pub fn is_retryable(&self) -> bool {+ /// Returns the delay before the given retry attempt, or `None` for a terminal error.+ ///+ /// The first retry is attempt one. Retryable errors use server advice when available and+ /// otherwise use exponential backoff with jitter. Callers enforce their own retry budgets.+ pub fn retry_delay(&self, retry_count: u64) -> Option<Duration> {match self.details() {CodexErrorDetails::TurnAborted| CodexErrorDetails::SessionBudgetExceeded@@ -397,7 +402,7 @@ impl CodexErr {| CodexErrorDetails::ServerOverloaded| CodexErrorDetails::CyberPolicy { .. }| CodexErrorDetails::BioPolicy { .. }- | CodexErrorDetails::MisalignmentPolicyViolation { .. } => false,+ | CodexErrorDetails::MisalignmentPolicyViolation { .. } => None,CodexErrorDetails::Stream(..)| CodexErrorDetails::RateLimitExceeded(_)| CodexErrorDetails::Timeout@@ -409,18 +414,22 @@ impl CodexErr {| CodexErrorDetails::InternalAgentDied| CodexErrorDetails::Io(_)| CodexErrorDetails::Json(_)- | CodexErrorDetails::TokioJoin(_) => true,+ | CodexErrorDetails::TokioJoin(_) => Some(+ self.server_retry_delay+ .unwrap_or_else(|| backoff(retry_count)),+ ),#[cfg(target_os = "linux")]- CodexErrorDetails::LandlockRuleset(_) | CodexErrorDetails::LandlockPathFd(_) => false,+ CodexErrorDetails::LandlockRuleset(_) | CodexErrorDetails::LandlockPathFd(_) => None,}}- pub fn retry_delay(&self) -> Option<Duration> {- self.retry_delay+ /// Returns only the delay advised by the server, without applying local retry policy.+ pub fn server_retry_delay(&self) -> Option<Duration> {+ self.server_retry_delay}pub fn with_retry_delay(mut self, retry_delay: Duration) -> Self {- self.retry_delay = Some(retry_delay);+ self.server_retry_delay = Some(retry_delay);self}diff --git a/codex-rs/protocol/src/error_tests.rs b/codex-rs/protocol/src/error_tests.rsindex 5357867ef0..921fb79ce7 100644--- a/codex-rs/protocol/src/error_tests.rs+++ b/codex-rs/protocol/src/error_tests.rs@@ -69,13 +69,45 @@ fn retryability_preserves_error_details_distinctions() {for (err, expected) in errors {assert_eq!(- err.is_retryable(),+ err.retry_delay(/*retry_count*/ 1).is_some(),expected,"unexpected retryability for {err:?}");}}+/// Retryable errors prefer server advice and otherwise back off by attempt; advice does not+/// make a terminal error retryable.+#[test]+fn retry_delay_distinguishes_server_advice_backoff_and_terminal_errors() {+ let error = CodexErr::InternalServerError;+ for (retry_count, expected_millis) in [(1, 180..220), (3, 720..880)] {+ let delay = error.retry_delay(retry_count).expect("retryable error");+ assert!(expected_millis.contains(&delay.as_millis()));+ }+ assert_eq!(error.server_retry_delay(), None);++ let advice = Duration::ZERO;+ let error = error.with_retry_delay(advice);+ assert_eq!(+ (+ error.retry_delay(/*retry_count*/ 1),+ error.retry_delay(/*retry_count*/ 3),+ error.server_retry_delay(),+ ),+ (Some(advice), Some(advice), Some(advice)),+ );++ let error = CodexErr::QuotaExceeded.with_retry_delay(advice);+ assert_eq!(+ (+ error.retry_delay(/*retry_count*/ 1),+ error.server_retry_delay(),+ ),+ (None, Some(advice)),+ );+}+fn rate_limit_snapshot() -> RateLimitSnapshot {let primary_reset_at = Utc.with_ymd_and_hms(2024, 1, 1, 1, 0, 0)