Lightweight coding agent that runs in your terminal
Rust
Honor Retry-After headers in failed Responses events (#50418)
Adam Perry @ OpenAI44dd77b71e88
diff --git a/codex-rs/codex-api/src/sse/responses.rs b/codex-rs/codex-api/src/sse/responses.rsindex 6b2a7ed47c..b9739289d1 100644--- a/codex-rs/codex-api/src/sse/responses.rs+++ b/codex-rs/codex-api/src/sse/responses.rs@@ -649,6 +649,10 @@ async fn process_sse_with_treatment(}}+#[cfg(test)]+#[path = "responses_error_tests.rs"]+mod error_tests;+#[cfg(test)]mod tests {use super::*;diff --git a/codex-rs/codex-api/src/sse/responses_error.rs b/codex-rs/codex-api/src/sse/responses_error.rsindex 872769c3b1..890f45ab6b 100644--- a/codex-rs/codex-api/src/sse/responses_error.rs+++ b/codex-rs/codex-api/src/sse/responses_error.rs@@ -1,8 +1,9 @@-//! Classifies failed Responses events and their rate-limit retry delays.+//! Classifies failed Responses events and their retry advice.//! Only rate-limit codes derive retry delays from plaintext messages.use crate::error::ApiError;use crate::error::parse_flex_unavailable;+use crate::responses_headers::json_headers_to_http_headers;use codex_http_client::RetryAfter;use codex_protocol::protocol::MisalignmentErrorDetails;use serde::Deserialize;@@ -23,19 +24,24 @@ struct Error {}pub(super) fn parse_failed_response(response: Option<Value>) -> ApiError {- if let Some(error) = response- .as_ref()- .and_then(|response| response.get("error"))- .and_then(parse_flex_unavailable)- {+ let error = response.as_ref().and_then(|response| response.get("error"));+ let retry_after_header = error+ .and_then(|error| error.get("headers"))+ .and_then(Value::as_object)+ .and_then(|headers| RetryAfter::from_headers(&json_headers_to_http_headers(headers)));+ if let Some(error) = error.and_then(parse_flex_unavailable) {return error;}- let Some(error) = response- .as_ref()- .and_then(|response| response.get("error"))- .and_then(|error| serde_json::from_value::<Error>(error.clone()).ok())+ let Some(error) = error.and_then(|error| serde_json::from_value::<Error>(error.clone()).ok())else {- return ApiError::Stream("response.failed event received".into());+ let message = "response.failed event received".to_string();+ return match retry_after_header {+ Some(_) => ApiError::Retryable {+ message,+ retry_after: retry_after_header,+ },+ None => ApiError::Stream(message),+ };};match error.code.as_deref() {@@ -78,9 +84,12 @@ pub(super) fn parse_failed_response(response: Option<Value>) -> ApiError {.message.unwrap_or_else(|| "Invalid request.".to_string()),},- Some("server_is_overloaded") => ApiError::ServerOverloaded { retry_after: None },+ Some("server_is_overloaded") => ApiError::ServerOverloaded {+ retry_after: retry_after_header,+ },Some("rate_limit_exceeded" | "slow_down") => {- let retry_after = try_parse_retry_delay(&error).and_then(RetryAfter::from_delay);+ let retry_after = retry_after_header+ .or_else(|| try_parse_retry_delay(&error).and_then(RetryAfter::from_delay));ApiError::RateLimitExceeded {message: error.message.unwrap_or_default(),retry_after,@@ -88,7 +97,7 @@ pub(super) fn parse_failed_response(response: Option<Value>) -> ApiError {}_ => ApiError::Retryable {message: error.message.unwrap_or_default(),- retry_after: None,+ retry_after: retry_after_header,},}}@@ -106,7 +115,7 @@ fn try_parse_retry_delay(err: &Error) -> Option<Duration> {let unit = unit.as_str().to_ascii_lowercase();if unit == "s" || unit.starts_with("second") {- return Some(Duration::from_secs_f64(value));+ return Duration::try_from_secs_f64(value).ok();} else if unit == "ms" {return Some(Duration::from_millis(value as u64));}diff --git a/codex-rs/codex-api/src/sse/responses_error_tests.rs b/codex-rs/codex-api/src/sse/responses_error_tests.rsnew file mode 100644index 0000000000..aeeb4cecac--- /dev/null+++ b/codex-rs/codex-api/src/sse/responses_error_tests.rs@@ -0,0 +1,156 @@+//! Regression coverage for retry advice carried by failed Responses events.++use super::process_responses_event;+use super::spawn_response_stream;+use crate::api_bridge::map_api_error;+use crate::common::ResponseEvent;+use bytes::Bytes;+use codex_client::StreamResponse;+use codex_http_client::RetryAfter;+use codex_protocol::protocol::CodexErrorInfo;+use futures::StreamExt;+use futures::stream;+use http::HeaderMap;+use http::StatusCode;+use pretty_assertions::assert_eq;+use serde_json::json;+use std::time::Duration;++/// Flex failures remain terminal even with retry headers and a later completed event.+#[tokio::test(start_paused = true)]+async fn flex_failure_with_retry_header_ends_stream_immediately() {+ let error = json!({+ "code": "flex_unavailable",+ "plan_type": 42,+ "headers": {"retry-after": "300"}+ });+ for failure in [+ json!({"type": "error", "error": error}),+ json!({"type": "response.failed", "response": {"error": error}}),+ ] {+ let completed = json!({"type": "response.completed", "response": {"id": "later"}});+ let bytes = stream::iter([Ok(Bytes::from(format!(+ "data: {failure}\n\ndata: {completed}\n\n"+ )))])+ .chain(stream::pending());+ let mut stream = spawn_response_stream(+ StreamResponse {+ status: StatusCode::OK,+ headers: HeaderMap::new(),+ bytes: Box::pin(bytes),+ },+ Duration::from_secs(60),+ /*telemetry*/ None,+ /*turn_state*/ None,+ );+ let error = loop {+ match stream.next().await {+ Some(Ok(ResponseEvent::RateLimits(_))) => {}+ Some(Err(error)) => break map_api_error(error),+ event => panic!("expected a Flex error, got {event:?}"),+ }+ };+ assert_eq!(+ (+ error.to_codex_protocol_error(),+ error.retry_delay(/*retry_count*/ 1)+ ),+ (CodexErrorInfo::FlexUnavailable, None)+ );+ assert!(stream.next().await.is_none());+ }+}++/// Error headers beat rate-limit message advice; invalid ones fall back.+#[tokio::test(start_paused = true)]+async fn rate_limit_plaintext_loses_to_valid_error_header() {+ for code in ["rate_limit_exceeded", "slow_down"] {+ for (error_header, expected_seconds) in [+ (Some("2"), 2),+ (Some("0"), 0),+ (Some("Wed, 21 Oct 2015 07:28:00 GMT"), 0),+ (Some("bad"), 35),+ (None, 35),+ ] {+ let event = serde_json::from_value(json!({+ "type": "response.failed",+ "response": {+ "error": {+ "code": code,+ "message": "Try again in 35 seconds.",+ "headers": {"Retry-After": error_header}+ }+ }+ }))+ .unwrap();+ let error = map_api_error(process_responses_event(event).unwrap_err().into_api_error());+ assert_eq!(+ error.retry_after(),+ RetryAfter::from_delay(Duration::from_secs(expected_seconds))+ );+ }+ }+}++/// Retry advice uses the same HTTP value validation as other event headers.+#[tokio::test(start_paused = true)]+async fn streamed_retry_after_uses_http_header_validation() {+ for (error_headers, expected_seconds) in [+ (json!({"retry-after": "\n5\n"}), 12),+ (json!({"retry-after": "\t5\t"}), 5),+ (json!({"Retry-After": "5", "retry-after": "30"}), 30),+ ] {+ let event = serde_json::from_value(json!({+ "type": "response.failed",+ "response": {"error": {"code": "rate_limit_exceeded", "message": "Try again in 12 seconds.", "headers": error_headers}}+ }))+ .unwrap();+ let error = map_api_error(process_responses_event(event).unwrap_err().into_api_error());+ assert_eq!(+ error.retry_after(),+ RetryAfter::from_delay(Duration::from_secs(expected_seconds))+ );+ }+}++/// Event headers retain their deadlines for overload and generic failures.+#[tokio::test(start_paused = true)]+async fn retry_header_survives_other_failed_responses() {+ for (code, expected_code) in [+ ("server_is_overloaded", CodexErrorInfo::ServerOverloaded),+ ("unknown_failure", CodexErrorInfo::Other),+ ] {+ let event = serde_json::from_value(json!({+ "type": "response.failed",+ "response": {"error": {"code": code, "headers": {"retry-after": "7"}}},+ }))+ .unwrap();+ let error = map_api_error(process_responses_event(event).unwrap_err().into_api_error());+ assert_eq!(+ (+ error.to_codex_protocol_error(),+ error.retry_after(),+ error.retry_delay(/*retry_count*/ 1),+ ),+ (+ expected_code,+ RetryAfter::from_delay(Duration::from_secs(7)),+ Some(Duration::from_secs(7)),+ )+ );+ }+}++/// An event header cannot make a quota or policy failure retryable.+#[tokio::test(start_paused = true)]+async fn retry_header_does_not_override_terminal_errors() {+ for code in ["insufficient_quota", "cyber_policy", "bio_policy"] {+ let event = serde_json::from_value(json!({+ "type": "response.failed",+ "response": {"error": {"code": code, "headers": {"retry-after": "1"}}}+ }))+ .unwrap();+ let error = map_api_error(process_responses_event(event).unwrap_err().into_api_error());+ assert_eq!(error.retry_delay(/*retry_count*/ 1), None, "{code}");+ }+}diff --git a/codex-rs/core/tests/suite/retry_after.rs b/codex-rs/core/tests/suite/retry_after.rsindex f69caf5208..bd15f5ac76 100644--- a/codex-rs/core/tests/suite/retry_after.rs+++ b/codex-rs/core/tests/suite/retry_after.rs@@ -686,7 +686,6 @@ async fn compact_v2_uses_retry_after() -> Result<()> {Ok(())}-// TODO(anp) respect Retry-After/// Remote compaction v2 stream failures retry without using the enclosing response header.#[tokio::test(flavor = "current_thread")]async fn compact_v2_stream_failure_uses_local_backoff_despite_retry_after() -> Result<()> {@@ -856,90 +855,26 @@ async fn compact_v2_stream_failure_without_retry_after_exhausts_stream_retries()Ok(())}-// TODO(anp) respect Retry-After-/// Remote compaction v2 already honors exact retry advice embedded in rate-limit messages.+/// Remote compaction honors message-only advice within its budget, with headers taking precedence.+#[test_case::test_case(Some("1"), "Try again in 4 seconds."; "nested_header")]+#[test_case::test_case(None, "Try again in 1 second."; "plaintext")]#[tokio::test(flavor = "current_thread")]-async fn compact_v2_rate_limit_message_uses_server_advised_retry_delay() -> Result<()> {- skip_if_no_network!(Ok(()));-- let mut telemetry = RetryTelemetryCapture::install();- let server = responses::start_mock_server().await;- let response_mock = responses::mount_response_sequence(- &server,- vec![- responses::sse_response(responses::sse(vec![- responses::ev_response_created("seed"),- responses::ev_completed("seed"),- ])),- responses::sse_response(responses::sse_failed(- "rate-limited",- "rate_limit_exceeded",- "Rate limit exceeded. Please try again in 1s.",- ))- .insert_header("Retry-After", "2"),- responses::sse_response(responses::sse(vec![- json!({- "type": "response.output_item.done",- "item": {- "type": "compaction",- "encrypted_content": "RETRIED_COMPACTION_SUMMARY",- }- }),- responses::ev_completed("compacted"),- ])),- ],- )- .await;- let test = test_codex()- .with_auth(CodexAuth::create_dummy_chatgpt_auth_for_testing())- .with_config(|config| {- config.model_provider.request_max_retries = Some(0);- config.model_provider.stream_max_retries = Some(1);- })- .build_with_auto_env(&server)- .await?;- test.submit_turn("seed history for compaction").await?;-- test.codex.submit(Op::Compact).await?;- let retry = telemetry.next_retry().await;- assert!(retry.delay <= Duration::from_secs(1));- assert_eq!(- retry,- RetryTelemetryEvent {- attempt: 1,- delay: retry.delay,- layer: "stream".into(),- operation: "remote_compaction_v2".into(),- }- );- assert!(wait_for_retry(&mut telemetry, &retry).await >= Duration::from_secs(1));- wait_for_turn_completion(&test).await;-- let requests = response_mock.requests();- assert_eq!(requests.len(), 3);- for request in &requests[1..] {- assert_eq!(request.path(), "/v1/responses");- assert!(- !request.inputs_of_type("compaction_trigger").is_empty(),- "expected a remote compaction v2 request"- );- }- assert_eq!(- telemetry.events.try_recv(),- Err(mpsc::error::TryRecvError::Empty)- );-- Ok(())-}--/// Remote compaction rate-limit messages provide exact retry advice without an HTTP header.-#[tokio::test(flavor = "current_thread")]-async fn compact_v2_rate_limit_message_without_retry_after_uses_server_advised_delay() -> Result<()>-{+async fn compact_v2_rate_limit_uses_server_advice(+ error_header: Option<&str>,+ message: &str,+) -> Result<()> {skip_if_no_network!(Ok(()));let mut telemetry = RetryTelemetryCapture::install();let server = responses::start_mock_server().await;+ let failed = responses::sse_response(responses::sse(vec![json!({+ "type": "response.failed",+ "response": {"error": {+ "code": "rate_limit_exceeded",+ "message": message,+ "headers": {"Retry-After": error_header}+ }}+ })]));let response_mock = responses::mount_response_sequence(&server,vec![@@ -947,11 +882,7 @@ async fn compact_v2_rate_limit_message_without_retry_after_uses_server_advised_dresponses::ev_response_created("seed"),responses::ev_completed("seed"),])),- responses::sse_response(responses::sse_failed(- "rate-limited",- "rate_limit_exceeded",- "Rate limit exceeded. Please try again in 1s.",- )),+ failed,responses::sse_response(responses::sse(vec![json!({"type": "response.output_item.done",@@ -1116,7 +1047,6 @@ async fn compact_v2_overload_without_retry_after_exhausts_request_retries() -> ROk(())}-// TODO(anp) respect Retry-After/// SSE failures currently retry with local backoff instead of the enclosing response header.#[tokio::test(flavor = "current_thread")]async fn sse_failure_uses_local_backoff_despite_retry_after() -> Result<()> {@@ -1272,79 +1202,35 @@ async fn sse_failure_without_retry_after_exhausts_stream_retries(code: &str) ->Ok(())}-/// Rate-limit messages already provide an exact retry delay without an HTTP header.-#[test_case::test_case("rate_limit_exceeded"; "rate_limit")]-#[test_case::test_case("slow_down"; "slow_down")]+/// Both sources survive to the agent's retry loop, and valid headers always beat message advice.+#[test_case::test_case("rate_limit_exceeded", Some("1"), "Rate limit exceeded. Try again in 4 seconds."; "azure_style_header")]+#[test_case::test_case("rate_limit_exceeded", None, "Rate limit exceeded. Try again in 1 second."; "azure_style_plaintext")]+#[test_case::test_case("slow_down", Some("1"), "Please try again in 4.000s."; "slow_down_fractional_header")]+#[test_case::test_case("slow_down", None, "Please try again in 1.000s."; "slow_down_fractional_plaintext")]+#[test_case::test_case("rate_limit_exceeded", Some("bad"), "Try again in 1 second."; "malformed_header")]+#[test_case::test_case("server_is_overloaded", Some("1"), "This model is overloaded."; "overload_header")]#[tokio::test(flavor = "current_thread")]-async fn sse_rate_limit_message_uses_server_advised_retry_delay(code: &str) -> Result<()> {- skip_if_no_network!(Ok(()));-- let mut telemetry = RetryTelemetryCapture::install();- let server = responses::start_mock_server().await;- let response_mock = responses::mount_response_sequence(- &server,- vec![- responses::sse_response(responses::sse_failed(- "rate-limited",- code,- "Rate limit exceeded. Please try again in 1s.",- )),- responses::sse_response(responses::sse(vec![- responses::ev_response_created("recovered"),- responses::ev_completed("recovered"),- ])),- ],- )- .await;- let test = test_codex()- .with_config(|config| {- config.model_provider.request_max_retries = Some(0);- config.model_provider.stream_max_retries = Some(1);- })- .build_with_auto_env(&server)- .await?;-- submit_user_input(&test, "retry after the rate-limit message delay").await?;- let retry = telemetry.next_retry().await;- assert!(retry.delay <= Duration::from_secs(1));- assert_eq!(- retry,- RetryTelemetryEvent {- attempt: 1,- delay: retry.delay,- layer: "stream".into(),- operation: "sampling".into(),- }- );- assert!(wait_for_retry(&mut telemetry, &retry).await >= Duration::from_secs(1));- wait_for_turn_completion(&test).await;-- assert_eq!(response_mock.requests().len(), 2);- assert_eq!(- telemetry.events.try_recv(),- Err(mpsc::error::TryRecvError::Empty)- );-- Ok(())-}--// TODO(anp) respect Retry-After-/// Rate-limit messages currently override an enclosing response's different retry delay.-#[tokio::test(flavor = "current_thread")]-async fn sse_rate_limit_message_with_retry_after_uses_server_advised_retry_delay() -> Result<()> {+async fn sse_failure_uses_server_advice(+ code: &str,+ error_header: Option<&str>,+ message: &str,+) -> Result<()> {skip_if_no_network!(Ok(()));let mut telemetry = RetryTelemetryCapture::install();let server = responses::start_mock_server().await;+ let failed = responses::sse_response(responses::sse(vec![json!({+ "type": "response.failed",+ "response": {"error": {+ "code": code,+ "message": message,+ "headers": {"Retry-After": error_header}+ }}+ })]));let response_mock = responses::mount_response_sequence(&server,vec![- responses::sse_response(responses::sse_failed(- "rate-limited",- "rate_limit_exceeded",- "Rate limit exceeded. Please try again in 1s.",- ))- .insert_header("Retry-After", "2"),+ failed,responses::sse_response(responses::sse(vec![responses::ev_response_created("recovered"),responses::ev_completed("recovered"),@@ -1360,7 +1246,7 @@ async fn sse_rate_limit_message_with_retry_after_uses_server_advised_retry_delay.build_with_auto_env(&server).await?;- submit_user_input(&test, "retry after both rate-limit delay signals").await?;+ submit_user_input(&test, "retry after the server's advice").await?;let retry = telemetry.next_retry().await;assert!(retry.delay <= Duration::from_secs(1));assert_eq!(@@ -1384,7 +1270,6 @@ async fn sse_rate_limit_message_with_retry_after_uses_server_advised_retry_delayOk(())}-// TODO(anp) respect Retry-After/// A streamed backend overload remains terminal despite an enclosing retry header.#[tokio::test(flavor = "current_thread")]async fn sse_overload_with_retry_after_is_terminal() -> Result<()> {@@ -1799,6 +1684,77 @@ async fn websocket_connection_limit_retries_with_local_backoff() -> Result<()> {Ok(())}+/// WebSocket failed-event headers take precedence over plaintext during an allowed retry.+#[tokio::test(flavor = "current_thread")]+async fn websocket_failed_event_uses_header_over_plaintext_retry_after() -> Result<()> {+ skip_if_no_network!(Ok(()));++ let mut telemetry = RetryTelemetryCapture::install();+ let server = responses::start_websocket_server(vec![+ vec![+ vec![+ responses::ev_response_created("prewarm"),+ responses::ev_completed("prewarm"),+ ],+ vec![json!({+ "type": "response.failed",+ "response": {+ "error": {+ "code": "rate_limit_exceeded",+ "message": "Rate limit exceeded. Try again in 4 seconds.",+ "headers": {"Retry-After": "1"}+ }+ }+ })],+ ],+ vec![vec![+ responses::ev_response_created("recovered"),+ responses::ev_completed("recovered"),+ ]],+ ])+ .await;+ let test = test_codex()+ .with_config(|config| {+ config.model_provider.request_max_retries = Some(0);+ config.model_provider.stream_max_retries = Some(1);+ })+ .build_with_websocket_server(&server)+ .await?;++ let warmup = server+ .wait_for_request(/*connection_index*/ 0, /*request_index*/ 0)+ .await;+ assert_eq!(warmup.body_json()["generate"].as_bool(), Some(false));+ submit_user_input(&test, "retry the rate-limited websocket stream").await?;++ let retry = telemetry.next_retry().await;+ assert!(retry.delay <= Duration::from_secs(1));+ assert_eq!(+ retry,+ RetryTelemetryEvent {+ attempt: 1,+ delay: retry.delay,+ layer: "stream".into(),+ operation: "sampling".into(),+ }+ );+ assert!(wait_for_retry(&mut telemetry, &retry).await >= Duration::from_secs(1));+ wait_for_turn_completion(&test).await;++ let connections = server.connections();+ assert_eq!(+ connections.iter().map(Vec::len).collect::<Vec<_>>(),+ vec![2, 1]+ );+ assert_eq!(+ telemetry.events.try_recv(),+ Err(mpsc::error::TryRecvError::Empty)+ );+ server.shutdown().await;++ Ok(())+}+/// Advised WebSocket failures reach HTTP after the existing retry budget. HTTP/// recovery uses its own retry budget and honors the preceding advice deadlines.#[tokio::test(flavor = "current_thread")]