Skip to content

armgit / codex

Public Imported from GitHub
0

Lightweight coding agent that runs in your terminal

Rust

Decouple HTTP retry backoff from overload integration testing (#41313)

Charlie Marsha73bf25d1780
diff --git a/codex-rs/core/tests/suite/retry_after.rs b/codex-rs/core/tests/suite/retry_after.rs
index 4cba8fcbc5..c24d510049 100644
--- a/codex-rs/core/tests/suite/retry_after.rs
+++ b/codex-rs/core/tests/suite/retry_after.rs
@@ -1,4 +1,9 @@
use anyhow::Result;
+use codex_client::RetryOn;
+use codex_client::RetryPolicy;
+use codex_client::run_with_retry;
+use codex_http_client::Request;
+use codex_http_client::TransportError;
use codex_login::CodexAuth;
use codex_protocol::protocol::CodexErrorInfo;
use codex_protocol::protocol::EventMsg;
@@ -10,6 +15,8 @@ use core_test_support::skip_if_no_network;
use core_test_support::test_codex::TestCodex;
use core_test_support::test_codex::test_codex;
use core_test_support::wait_for_event;
+use http::Method;
+use http::StatusCode;
use pretty_assertions::assert_eq;
use serde_json::json;
use std::net::TcpListener;
@@ -182,10 +189,9 @@ impl RetryTelemetryCapture {
}
async fn next_retry(&mut self) -> RetryTelemetryEvent {
- let retry = self
- .events
- .recv()
+ let retry = tokio::time::timeout(Duration::from_secs(10), self.events.recv())
.await
+ .expect("timed out waiting for retry telemetry")
.expect("retry telemetry subscriber should remain installed");
// Parallel tests may first register request callsites without our thread-local subscriber.
tracing::callsite::rebuild_interest_cache();
@@ -197,11 +203,13 @@ async fn wait_for_retry(
telemetry: &mut RetryTelemetryCapture,
retry: &RetryTelemetryEvent,
) -> Duration {
- let elapsed = telemetry
- .resumptions
- .recv()
- .await
- .expect("retry should start another request after its sleep");
+ let elapsed = tokio::time::timeout(
+ retry.delay + Duration::from_secs(10),
+ telemetry.resumptions.recv(),
+ )
+ .await
+ .expect("timed out waiting for the request after a retry")
+ .expect("retry should start another request after its sleep");
assert!(
elapsed >= retry.delay,
"{} {} retry waited {elapsed:?}, less than its selected {:?} delay",
@@ -285,12 +293,63 @@ async fn responses_http_uses_local_backoff_despite_retry_after() -> Result<()> {
Ok(())
}
+/// Check backoff without a live server or tracing events coordinating the retry loop.
+#[tokio::test(start_paused = true)]
+async fn http_retry_backoff_exhausts_attempts() {
+ let attempts = Mutex::new(Vec::new());
+ let result = run_with_retry(
+ RetryPolicy {
+ max_attempts: 2,
+ base_delay: Duration::from_millis(200),
+ retry_on: RetryOn {
+ retry_429: false,
+ retry_5xx: true,
+ retry_transport: false,
+ },
+ },
+ || Request::new(Method::POST, "http://localhost/v1/responses".into()),
+ |_, attempt| {
+ attempts
+ .lock()
+ .expect("retry attempts should not be poisoned")
+ .push((attempt, tokio::time::Instant::now()));
+ std::future::ready(Err::<(), _>(TransportError::Http {
+ status: StatusCode::SERVICE_UNAVAILABLE,
+ url: None,
+ headers: None,
+ body: None,
+ }))
+ },
+ )
+ .await;
+
+ assert!(
+ matches!(result, Err(TransportError::Http { status, .. }) if status == StatusCode::SERVICE_UNAVAILABLE)
+ );
+ let attempts = attempts
+ .into_inner()
+ .expect("retry attempts should not be poisoned");
+ assert_eq!(
+ attempts
+ .iter()
+ .map(|(attempt, _)| *attempt)
+ .collect::<Vec<_>>(),
+ vec![0, 1, 2]
+ );
+ assert!(
+ (FIRST_RETRY_MIN_DELAY..=FIRST_RETRY_MAX_DELAY).contains(&(attempts[1].1 - attempts[0].1))
+ );
+ assert!(
+ (SECOND_RETRY_MIN_DELAY..=SECOND_RETRY_MAX_DELAY)
+ .contains(&(attempts[2].1 - attempts[1].1))
+ );
+}
+
/// Headerless HTTP overloads currently exhaust request retries before emitting one terminal error.
#[tokio::test(flavor = "current_thread")]
async fn responses_http_overload_without_retry_after_exhausts_request_retries() -> Result<()> {
skip_if_no_network!(Ok(()));
- let mut telemetry = RetryTelemetryCapture::install();
let server = responses::start_mock_server().await;
Mock::given(method("POST"))
.and(path("/v1/responses"))
@@ -309,57 +368,36 @@ async fn responses_http_overload_without_retry_after_exhausts_request_retries()
.await?;
submit_user_input(&test, "reject the disabled model").await?;
- let first_retry = telemetry.next_retry().await;
- assert!((FIRST_RETRY_MIN_DELAY..FIRST_RETRY_MAX_DELAY).contains(&first_retry.delay));
- assert_eq!(
- first_retry,
- RetryTelemetryEvent {
- attempt: 1,
- delay: first_retry.delay,
- layer: "http".into(),
- operation: "request".into(),
- }
- );
- wait_for_retry(&mut telemetry, &first_retry).await;
- let second_retry = telemetry.next_retry().await;
- assert!((SECOND_RETRY_MIN_DELAY..SECOND_RETRY_MAX_DELAY).contains(&second_retry.delay));
- assert_eq!(
- second_retry,
- RetryTelemetryEvent {
- attempt: 2,
- delay: second_retry.delay,
- layer: "http".into(),
- operation: "request".into(),
- }
- );
- wait_for_retry(&mut telemetry, &second_retry).await;
-
let mut error_events = 0;
let mut stream_error_events = 0;
- loop {
- match wait_for_event(&test.codex, |_| true).await {
- EventMsg::Error(error) => {
- error_events += 1;
- assert_eq!(
- error.codex_error_info,
- Some(CodexErrorInfo::ServerOverloaded)
- );
- assert_eq!(
- error.message,
- "Selected model is at capacity. Please try a different model."
- );
- }
- EventMsg::StreamError(_) => stream_error_events += 1,
- EventMsg::TurnComplete(event) => {
- assert_eq!(
- event.error.and_then(|error| error.codex_error_info),
- Some(CodexErrorInfo::ServerOverloaded)
- );
- break;
+ tokio::time::timeout(Duration::from_secs(10), async {
+ loop {
+ match wait_for_event(&test.codex, |_| true).await {
+ EventMsg::Error(error) => {
+ error_events += 1;
+ assert_eq!(
+ error.codex_error_info,
+ Some(CodexErrorInfo::ServerOverloaded)
+ );
+ assert_eq!(
+ error.message,
+ "Selected model is at capacity. Please try a different model."
+ );
+ }
+ EventMsg::StreamError(_) => stream_error_events += 1,
+ EventMsg::TurnComplete(event) => {
+ assert_eq!(
+ event.error.and_then(|error| error.codex_error_info),
+ Some(CodexErrorInfo::ServerOverloaded)
+ );
+ break;
+ }
+ _ => {}
}
- _ => {}
}
- }
+ })
+ .await
+ .expect("overload retries should finish the turn within 10 seconds");
assert_eq!(error_events, 1);
assert_eq!(stream_error_events, 0);
@@ -374,10 +412,6 @@ async fn responses_http_overload_without_retry_after_exhausts_request_retries()
request_count, 3,
"headerless overload should exhaust the configured request retries"
);
- assert_eq!(
- telemetry.events.try_recv(),
- Err(mpsc::error::TryRecvError::Empty)
- );
Ok(())
}