Skip to content

armgit / codex

Public Imported from GitHub
0

Lightweight coding agent that runs in your terminal

Rust

Honor Retry-After headers in failed Responses events (#50418)

Adam Perry @ OpenAI44dd77b71e88
diff --git a/codex-rs/codex-api/src/sse/responses.rs b/codex-rs/codex-api/src/sse/responses.rs
index 6b2a7ed47c..b9739289d1 100644
--- a/codex-rs/codex-api/src/sse/responses.rs
+++ b/codex-rs/codex-api/src/sse/responses.rs
@@ -649,6 +649,10 @@ async fn process_sse_with_treatment(
}
}
+#[cfg(test)]
+#[path = "responses_error_tests.rs"]
+mod error_tests;
+
#[cfg(test)]
mod tests {
use super::*;
diff --git a/codex-rs/codex-api/src/sse/responses_error.rs b/codex-rs/codex-api/src/sse/responses_error.rs
index 872769c3b1..890f45ab6b 100644
--- a/codex-rs/codex-api/src/sse/responses_error.rs
+++ b/codex-rs/codex-api/src/sse/responses_error.rs
@@ -1,8 +1,9 @@
-//! Classifies failed Responses events and their rate-limit retry delays.
+//! Classifies failed Responses events and their retry advice.
//! Only rate-limit codes derive retry delays from plaintext messages.
use crate::error::ApiError;
use crate::error::parse_flex_unavailable;
+use crate::responses_headers::json_headers_to_http_headers;
use codex_http_client::RetryAfter;
use codex_protocol::protocol::MisalignmentErrorDetails;
use serde::Deserialize;
@@ -23,19 +24,24 @@ struct Error {
}
pub(super) fn parse_failed_response(response: Option<Value>) -> ApiError {
- if let Some(error) = response
- .as_ref()
- .and_then(|response| response.get("error"))
- .and_then(parse_flex_unavailable)
- {
+ let error = response.as_ref().and_then(|response| response.get("error"));
+ let retry_after_header = error
+ .and_then(|error| error.get("headers"))
+ .and_then(Value::as_object)
+ .and_then(|headers| RetryAfter::from_headers(&json_headers_to_http_headers(headers)));
+ if let Some(error) = error.and_then(parse_flex_unavailable) {
return error;
}
- let Some(error) = response
- .as_ref()
- .and_then(|response| response.get("error"))
- .and_then(|error| serde_json::from_value::<Error>(error.clone()).ok())
+ let Some(error) = error.and_then(|error| serde_json::from_value::<Error>(error.clone()).ok())
else {
- return ApiError::Stream("response.failed event received".into());
+ let message = "response.failed event received".to_string();
+ return match retry_after_header {
+ Some(_) => ApiError::Retryable {
+ message,
+ retry_after: retry_after_header,
+ },
+ None => ApiError::Stream(message),
+ };
};
match error.code.as_deref() {
@@ -78,9 +84,12 @@ pub(super) fn parse_failed_response(response: Option<Value>) -> ApiError {
.message
.unwrap_or_else(|| "Invalid request.".to_string()),
},
- Some("server_is_overloaded") => ApiError::ServerOverloaded { retry_after: None },
+ Some("server_is_overloaded") => ApiError::ServerOverloaded {
+ retry_after: retry_after_header,
+ },
Some("rate_limit_exceeded" | "slow_down") => {
- let retry_after = try_parse_retry_delay(&error).and_then(RetryAfter::from_delay);
+ let retry_after = retry_after_header
+ .or_else(|| try_parse_retry_delay(&error).and_then(RetryAfter::from_delay));
ApiError::RateLimitExceeded {
message: error.message.unwrap_or_default(),
retry_after,
@@ -88,7 +97,7 @@ pub(super) fn parse_failed_response(response: Option<Value>) -> ApiError {
}
_ => ApiError::Retryable {
message: error.message.unwrap_or_default(),
- retry_after: None,
+ retry_after: retry_after_header,
},
}
}
@@ -106,7 +115,7 @@ fn try_parse_retry_delay(err: &Error) -> Option<Duration> {
let unit = unit.as_str().to_ascii_lowercase();
if unit == "s" || unit.starts_with("second") {
- return Some(Duration::from_secs_f64(value));
+ return Duration::try_from_secs_f64(value).ok();
} else if unit == "ms" {
return Some(Duration::from_millis(value as u64));
}
diff --git a/codex-rs/codex-api/src/sse/responses_error_tests.rs b/codex-rs/codex-api/src/sse/responses_error_tests.rs
new file mode 100644
index 0000000000..aeeb4cecac
--- /dev/null
+++ b/codex-rs/codex-api/src/sse/responses_error_tests.rs
@@ -0,0 +1,156 @@
+//! Regression coverage for retry advice carried by failed Responses events.
+
+use super::process_responses_event;
+use super::spawn_response_stream;
+use crate::api_bridge::map_api_error;
+use crate::common::ResponseEvent;
+use bytes::Bytes;
+use codex_client::StreamResponse;
+use codex_http_client::RetryAfter;
+use codex_protocol::protocol::CodexErrorInfo;
+use futures::StreamExt;
+use futures::stream;
+use http::HeaderMap;
+use http::StatusCode;
+use pretty_assertions::assert_eq;
+use serde_json::json;
+use std::time::Duration;
+
+/// Flex failures remain terminal even with retry headers and a later completed event.
+#[tokio::test(start_paused = true)]
+async fn flex_failure_with_retry_header_ends_stream_immediately() {
+ let error = json!({
+ "code": "flex_unavailable",
+ "plan_type": 42,
+ "headers": {"retry-after": "300"}
+ });
+ for failure in [
+ json!({"type": "error", "error": error}),
+ json!({"type": "response.failed", "response": {"error": error}}),
+ ] {
+ let completed = json!({"type": "response.completed", "response": {"id": "later"}});
+ let bytes = stream::iter([Ok(Bytes::from(format!(
+ "data: {failure}\n\ndata: {completed}\n\n"
+ )))])
+ .chain(stream::pending());
+ let mut stream = spawn_response_stream(
+ StreamResponse {
+ status: StatusCode::OK,
+ headers: HeaderMap::new(),
+ bytes: Box::pin(bytes),
+ },
+ Duration::from_secs(60),
+ /*telemetry*/ None,
+ /*turn_state*/ None,
+ );
+ let error = loop {
+ match stream.next().await {
+ Some(Ok(ResponseEvent::RateLimits(_))) => {}
+ Some(Err(error)) => break map_api_error(error),
+ event => panic!("expected a Flex error, got {event:?}"),
+ }
+ };
+ assert_eq!(
+ (
+ error.to_codex_protocol_error(),
+ error.retry_delay(/*retry_count*/ 1)
+ ),
+ (CodexErrorInfo::FlexUnavailable, None)
+ );
+ assert!(stream.next().await.is_none());
+ }
+}
+
+/// Error headers beat rate-limit message advice; invalid ones fall back.
+#[tokio::test(start_paused = true)]
+async fn rate_limit_plaintext_loses_to_valid_error_header() {
+ for code in ["rate_limit_exceeded", "slow_down"] {
+ for (error_header, expected_seconds) in [
+ (Some("2"), 2),
+ (Some("0"), 0),
+ (Some("Wed, 21 Oct 2015 07:28:00 GMT"), 0),
+ (Some("bad"), 35),
+ (None, 35),
+ ] {
+ let event = serde_json::from_value(json!({
+ "type": "response.failed",
+ "response": {
+ "error": {
+ "code": code,
+ "message": "Try again in 35 seconds.",
+ "headers": {"Retry-After": error_header}
+ }
+ }
+ }))
+ .unwrap();
+ let error = map_api_error(process_responses_event(event).unwrap_err().into_api_error());
+ assert_eq!(
+ error.retry_after(),
+ RetryAfter::from_delay(Duration::from_secs(expected_seconds))
+ );
+ }
+ }
+}
+
+/// Retry advice uses the same HTTP value validation as other event headers.
+#[tokio::test(start_paused = true)]
+async fn streamed_retry_after_uses_http_header_validation() {
+ for (error_headers, expected_seconds) in [
+ (json!({"retry-after": "\n5\n"}), 12),
+ (json!({"retry-after": "\t5\t"}), 5),
+ (json!({"Retry-After": "5", "retry-after": "30"}), 30),
+ ] {
+ let event = serde_json::from_value(json!({
+ "type": "response.failed",
+ "response": {"error": {"code": "rate_limit_exceeded", "message": "Try again in 12 seconds.", "headers": error_headers}}
+ }))
+ .unwrap();
+ let error = map_api_error(process_responses_event(event).unwrap_err().into_api_error());
+ assert_eq!(
+ error.retry_after(),
+ RetryAfter::from_delay(Duration::from_secs(expected_seconds))
+ );
+ }
+}
+
+/// Event headers retain their deadlines for overload and generic failures.
+#[tokio::test(start_paused = true)]
+async fn retry_header_survives_other_failed_responses() {
+ for (code, expected_code) in [
+ ("server_is_overloaded", CodexErrorInfo::ServerOverloaded),
+ ("unknown_failure", CodexErrorInfo::Other),
+ ] {
+ let event = serde_json::from_value(json!({
+ "type": "response.failed",
+ "response": {"error": {"code": code, "headers": {"retry-after": "7"}}},
+ }))
+ .unwrap();
+ let error = map_api_error(process_responses_event(event).unwrap_err().into_api_error());
+ assert_eq!(
+ (
+ error.to_codex_protocol_error(),
+ error.retry_after(),
+ error.retry_delay(/*retry_count*/ 1),
+ ),
+ (
+ expected_code,
+ RetryAfter::from_delay(Duration::from_secs(7)),
+ Some(Duration::from_secs(7)),
+ )
+ );
+ }
+}
+
+/// An event header cannot make a quota or policy failure retryable.
+#[tokio::test(start_paused = true)]
+async fn retry_header_does_not_override_terminal_errors() {
+ for code in ["insufficient_quota", "cyber_policy", "bio_policy"] {
+ let event = serde_json::from_value(json!({
+ "type": "response.failed",
+ "response": {"error": {"code": code, "headers": {"retry-after": "1"}}}
+ }))
+ .unwrap();
+ let error = map_api_error(process_responses_event(event).unwrap_err().into_api_error());
+ assert_eq!(error.retry_delay(/*retry_count*/ 1), None, "{code}");
+ }
+}
diff --git a/codex-rs/core/tests/suite/retry_after.rs b/codex-rs/core/tests/suite/retry_after.rs
index f69caf5208..bd15f5ac76 100644
--- a/codex-rs/core/tests/suite/retry_after.rs
+++ b/codex-rs/core/tests/suite/retry_after.rs
@@ -686,7 +686,6 @@ async fn compact_v2_uses_retry_after() -> Result<()> {
Ok(())
}
-// TODO(anp) respect Retry-After
/// Remote compaction v2 stream failures retry without using the enclosing response header.
#[tokio::test(flavor = "current_thread")]
async fn compact_v2_stream_failure_uses_local_backoff_despite_retry_after() -> Result<()> {
@@ -856,90 +855,26 @@ async fn compact_v2_stream_failure_without_retry_after_exhausts_stream_retries()
Ok(())
}
-// TODO(anp) respect Retry-After
-/// Remote compaction v2 already honors exact retry advice embedded in rate-limit messages.
+/// Remote compaction honors message-only advice within its budget, with headers taking precedence.
+#[test_case::test_case(Some("1"), "Try again in 4 seconds."; "nested_header")]
+#[test_case::test_case(None, "Try again in 1 second."; "plaintext")]
#[tokio::test(flavor = "current_thread")]
-async fn compact_v2_rate_limit_message_uses_server_advised_retry_delay() -> Result<()> {
- skip_if_no_network!(Ok(()));
-
- let mut telemetry = RetryTelemetryCapture::install();
- let server = responses::start_mock_server().await;
- let response_mock = responses::mount_response_sequence(
- &server,
- vec![
- responses::sse_response(responses::sse(vec![
- responses::ev_response_created("seed"),
- responses::ev_completed("seed"),
- ])),
- responses::sse_response(responses::sse_failed(
- "rate-limited",
- "rate_limit_exceeded",
- "Rate limit exceeded. Please try again in 1s.",
- ))
- .insert_header("Retry-After", "2"),
- responses::sse_response(responses::sse(vec![
- json!({
- "type": "response.output_item.done",
- "item": {
- "type": "compaction",
- "encrypted_content": "RETRIED_COMPACTION_SUMMARY",
- }
- }),
- responses::ev_completed("compacted"),
- ])),
- ],
- )
- .await;
- let test = test_codex()
- .with_auth(CodexAuth::create_dummy_chatgpt_auth_for_testing())
- .with_config(|config| {
- config.model_provider.request_max_retries = Some(0);
- config.model_provider.stream_max_retries = Some(1);
- })
- .build_with_auto_env(&server)
- .await?;
- test.submit_turn("seed history for compaction").await?;
-
- test.codex.submit(Op::Compact).await?;
- let retry = telemetry.next_retry().await;
- assert!(retry.delay <= Duration::from_secs(1));
- assert_eq!(
- retry,
- RetryTelemetryEvent {
- attempt: 1,
- delay: retry.delay,
- layer: "stream".into(),
- operation: "remote_compaction_v2".into(),
- }
- );
- assert!(wait_for_retry(&mut telemetry, &retry).await >= Duration::from_secs(1));
- wait_for_turn_completion(&test).await;
-
- let requests = response_mock.requests();
- assert_eq!(requests.len(), 3);
- for request in &requests[1..] {
- assert_eq!(request.path(), "/v1/responses");
- assert!(
- !request.inputs_of_type("compaction_trigger").is_empty(),
- "expected a remote compaction v2 request"
- );
- }
- assert_eq!(
- telemetry.events.try_recv(),
- Err(mpsc::error::TryRecvError::Empty)
- );
-
- Ok(())
-}
-
-/// Remote compaction rate-limit messages provide exact retry advice without an HTTP header.
-#[tokio::test(flavor = "current_thread")]
-async fn compact_v2_rate_limit_message_without_retry_after_uses_server_advised_delay() -> Result<()>
-{
+async fn compact_v2_rate_limit_uses_server_advice(
+ error_header: Option<&str>,
+ message: &str,
+) -> Result<()> {
skip_if_no_network!(Ok(()));
let mut telemetry = RetryTelemetryCapture::install();
let server = responses::start_mock_server().await;
+ let failed = responses::sse_response(responses::sse(vec![json!({
+ "type": "response.failed",
+ "response": {"error": {
+ "code": "rate_limit_exceeded",
+ "message": message,
+ "headers": {"Retry-After": error_header}
+ }}
+ })]));
let response_mock = responses::mount_response_sequence(
&server,
vec![
@@ -947,11 +882,7 @@ async fn compact_v2_rate_limit_message_without_retry_after_uses_server_advised_d
responses::ev_response_created("seed"),
responses::ev_completed("seed"),
])),
- responses::sse_response(responses::sse_failed(
- "rate-limited",
- "rate_limit_exceeded",
- "Rate limit exceeded. Please try again in 1s.",
- )),
+ failed,
responses::sse_response(responses::sse(vec![
json!({
"type": "response.output_item.done",
@@ -1116,7 +1047,6 @@ async fn compact_v2_overload_without_retry_after_exhausts_request_retries() -> R
Ok(())
}
-// TODO(anp) respect Retry-After
/// SSE failures currently retry with local backoff instead of the enclosing response header.
#[tokio::test(flavor = "current_thread")]
async fn sse_failure_uses_local_backoff_despite_retry_after() -> Result<()> {
@@ -1272,79 +1202,35 @@ async fn sse_failure_without_retry_after_exhausts_stream_retries(code: &str) ->
Ok(())
}
-/// Rate-limit messages already provide an exact retry delay without an HTTP header.
-#[test_case::test_case("rate_limit_exceeded"; "rate_limit")]
-#[test_case::test_case("slow_down"; "slow_down")]
+/// Both sources survive to the agent's retry loop, and valid headers always beat message advice.
+#[test_case::test_case("rate_limit_exceeded", Some("1"), "Rate limit exceeded. Try again in 4 seconds."; "azure_style_header")]
+#[test_case::test_case("rate_limit_exceeded", None, "Rate limit exceeded. Try again in 1 second."; "azure_style_plaintext")]
+#[test_case::test_case("slow_down", Some("1"), "Please try again in 4.000s."; "slow_down_fractional_header")]
+#[test_case::test_case("slow_down", None, "Please try again in 1.000s."; "slow_down_fractional_plaintext")]
+#[test_case::test_case("rate_limit_exceeded", Some("bad"), "Try again in 1 second."; "malformed_header")]
+#[test_case::test_case("server_is_overloaded", Some("1"), "This model is overloaded."; "overload_header")]
#[tokio::test(flavor = "current_thread")]
-async fn sse_rate_limit_message_uses_server_advised_retry_delay(code: &str) -> Result<()> {
- skip_if_no_network!(Ok(()));
-
- let mut telemetry = RetryTelemetryCapture::install();
- let server = responses::start_mock_server().await;
- let response_mock = responses::mount_response_sequence(
- &server,
- vec![
- responses::sse_response(responses::sse_failed(
- "rate-limited",
- code,
- "Rate limit exceeded. Please try again in 1s.",
- )),
- responses::sse_response(responses::sse(vec![
- responses::ev_response_created("recovered"),
- responses::ev_completed("recovered"),
- ])),
- ],
- )
- .await;
- let test = test_codex()
- .with_config(|config| {
- config.model_provider.request_max_retries = Some(0);
- config.model_provider.stream_max_retries = Some(1);
- })
- .build_with_auto_env(&server)
- .await?;
-
- submit_user_input(&test, "retry after the rate-limit message delay").await?;
- let retry = telemetry.next_retry().await;
- assert!(retry.delay <= Duration::from_secs(1));
- assert_eq!(
- retry,
- RetryTelemetryEvent {
- attempt: 1,
- delay: retry.delay,
- layer: "stream".into(),
- operation: "sampling".into(),
- }
- );
- assert!(wait_for_retry(&mut telemetry, &retry).await >= Duration::from_secs(1));
- wait_for_turn_completion(&test).await;
-
- assert_eq!(response_mock.requests().len(), 2);
- assert_eq!(
- telemetry.events.try_recv(),
- Err(mpsc::error::TryRecvError::Empty)
- );
-
- Ok(())
-}
-
-// TODO(anp) respect Retry-After
-/// Rate-limit messages currently override an enclosing response's different retry delay.
-#[tokio::test(flavor = "current_thread")]
-async fn sse_rate_limit_message_with_retry_after_uses_server_advised_retry_delay() -> Result<()> {
+async fn sse_failure_uses_server_advice(
+ code: &str,
+ error_header: Option<&str>,
+ message: &str,
+) -> Result<()> {
skip_if_no_network!(Ok(()));
let mut telemetry = RetryTelemetryCapture::install();
let server = responses::start_mock_server().await;
+ let failed = responses::sse_response(responses::sse(vec![json!({
+ "type": "response.failed",
+ "response": {"error": {
+ "code": code,
+ "message": message,
+ "headers": {"Retry-After": error_header}
+ }}
+ })]));
let response_mock = responses::mount_response_sequence(
&server,
vec![
- responses::sse_response(responses::sse_failed(
- "rate-limited",
- "rate_limit_exceeded",
- "Rate limit exceeded. Please try again in 1s.",
- ))
- .insert_header("Retry-After", "2"),
+ failed,
responses::sse_response(responses::sse(vec![
responses::ev_response_created("recovered"),
responses::ev_completed("recovered"),
@@ -1360,7 +1246,7 @@ async fn sse_rate_limit_message_with_retry_after_uses_server_advised_retry_delay
.build_with_auto_env(&server)
.await?;
- submit_user_input(&test, "retry after both rate-limit delay signals").await?;
+ submit_user_input(&test, "retry after the server's advice").await?;
let retry = telemetry.next_retry().await;
assert!(retry.delay <= Duration::from_secs(1));
assert_eq!(
@@ -1384,7 +1270,6 @@ async fn sse_rate_limit_message_with_retry_after_uses_server_advised_retry_delay
Ok(())
}
-// TODO(anp) respect Retry-After
/// A streamed backend overload remains terminal despite an enclosing retry header.
#[tokio::test(flavor = "current_thread")]
async fn sse_overload_with_retry_after_is_terminal() -> Result<()> {
@@ -1799,6 +1684,77 @@ async fn websocket_connection_limit_retries_with_local_backoff() -> Result<()> {
Ok(())
}
+/// WebSocket failed-event headers take precedence over plaintext during an allowed retry.
+#[tokio::test(flavor = "current_thread")]
+async fn websocket_failed_event_uses_header_over_plaintext_retry_after() -> Result<()> {
+ skip_if_no_network!(Ok(()));
+
+ let mut telemetry = RetryTelemetryCapture::install();
+ let server = responses::start_websocket_server(vec![
+ vec![
+ vec![
+ responses::ev_response_created("prewarm"),
+ responses::ev_completed("prewarm"),
+ ],
+ vec![json!({
+ "type": "response.failed",
+ "response": {
+ "error": {
+ "code": "rate_limit_exceeded",
+ "message": "Rate limit exceeded. Try again in 4 seconds.",
+ "headers": {"Retry-After": "1"}
+ }
+ }
+ })],
+ ],
+ vec![vec![
+ responses::ev_response_created("recovered"),
+ responses::ev_completed("recovered"),
+ ]],
+ ])
+ .await;
+ let test = test_codex()
+ .with_config(|config| {
+ config.model_provider.request_max_retries = Some(0);
+ config.model_provider.stream_max_retries = Some(1);
+ })
+ .build_with_websocket_server(&server)
+ .await?;
+
+ let warmup = server
+ .wait_for_request(/*connection_index*/ 0, /*request_index*/ 0)
+ .await;
+ assert_eq!(warmup.body_json()["generate"].as_bool(), Some(false));
+ submit_user_input(&test, "retry the rate-limited websocket stream").await?;
+
+ let retry = telemetry.next_retry().await;
+ assert!(retry.delay <= Duration::from_secs(1));
+ assert_eq!(
+ retry,
+ RetryTelemetryEvent {
+ attempt: 1,
+ delay: retry.delay,
+ layer: "stream".into(),
+ operation: "sampling".into(),
+ }
+ );
+ assert!(wait_for_retry(&mut telemetry, &retry).await >= Duration::from_secs(1));
+ wait_for_turn_completion(&test).await;
+
+ let connections = server.connections();
+ assert_eq!(
+ connections.iter().map(Vec::len).collect::<Vec<_>>(),
+ vec![2, 1]
+ );
+ assert_eq!(
+ telemetry.events.try_recv(),
+ Err(mpsc::error::TryRecvError::Empty)
+ );
+ server.shutdown().await;
+
+ Ok(())
+}
+
/// Advised WebSocket failures reach HTTP after the existing retry budget. HTTP
/// recovery uses its own retry budget and honors the preceding advice deadlines.
#[tokio::test(flavor = "current_thread")]