Lightweight coding agent that runs in your terminal
Rust
Back off automatic remote control reconnects with jitter (#50348)
cooper-oai9d2b60303e83
diff --git a/codex-rs/app-server-transport/src/transport/remote_control/tests.rs b/codex-rs/app-server-transport/src/transport/remote_control/tests.rsindex 2337b2a658..611d540583 100644--- a/codex-rs/app-server-transport/src/transport/remote_control/tests.rs+++ b/codex-rs/app-server-transport/src/transport/remote_control/tests.rs@@ -1136,7 +1136,8 @@ async fn remote_control_transport_reconnects_after_disconnect() {drop(first_websocket);let (second_handshake_request, mut second_websocket) =- accept_remote_control_backend_connection(&listener).await;+ accept_remote_control_backend_connection_with_timeout(&listener, Duration::from_secs(45))+ .await;assert_eq!(second_handshake_request.headers.get("authorization"),Some(&format!("Bearer {TEST_REMOTE_CONTROL_SERVER_TOKEN}"))@@ -1237,7 +1238,8 @@ async fn remote_control_transport_refreshes_server_token_after_websocket_unautho);respond_with_status(websocket_request.stream, "401 Unauthorized", "").await;- let refresh_request = accept_http_request(&listener).await;+ let refresh_request =+ accept_http_request_with_timeout(&listener, Duration::from_secs(45)).await;assert_eq!(refresh_request.request_line,"POST /backend-api/wham/remote/control/server/refresh HTTP/1.1"@@ -1682,7 +1684,8 @@ async fn remote_control_transport_clears_outgoing_buffer_when_backend_acks() {.expect("first websocket should close");drop(first_websocket);- let mut second_websocket = accept_remote_control_connection(&listener).await;+ let mut second_websocket =+ accept_remote_control_connection_with_timeout(&listener, Duration::from_secs(45)).await;send_client_event(&mut second_websocket,ClientEnvelope {@@ -2713,7 +2716,8 @@ async fn remote_control_http_mode_preserves_stale_enrollment_when_reenrollment_f);respond_with_status(enroll_request.stream, "500 Internal Server Error", "failed").await;- let retry_refresh_request = accept_http_request(&listener).await;+ let retry_refresh_request =+ accept_http_request_with_timeout(&listener, Duration::from_secs(45)).await;assert_eq!(retry_refresh_request.request_line,"POST /backend-api/wham/remote/control/server/refresh HTTP/1.1"@@ -2880,7 +2884,9 @@ async fn remote_control_http_mode_preserves_enrollment_after_generic_websocket_4Some(stale_enrollment.clone()));- let (handshake_request, _websocket) = accept_remote_control_backend_connection(&listener).await;+ let (handshake_request, _websocket) =+ accept_remote_control_backend_connection_with_timeout(&listener, Duration::from_secs(45))+ .await;assert_eq!(handshake_request.headers.get("x-codex-server-id"),Some(&stale_enrollment.server_id)@@ -2946,7 +2952,14 @@ struct CapturedWebSocketRequest {}async fn accept_remote_control_connection(listener: &TcpListener) -> WebSocketStream<TcpStream> {- let (stream, _) = timeout(Duration::from_secs(5), listener.accept())+ accept_remote_control_connection_with_timeout(listener, Duration::from_secs(5)).await+}++async fn accept_remote_control_connection_with_timeout(+ listener: &TcpListener,+ accept_timeout: Duration,+) -> WebSocketStream<TcpStream> {+ let (stream, _) = timeout(accept_timeout, listener.accept()).await.expect("remote control should connect in time").expect("listener accept should succeed");@@ -2956,7 +2969,14 @@ async fn accept_remote_control_connection(listener: &TcpListener) -> WebSocketSt}async fn accept_http_request(listener: &TcpListener) -> CapturedHttpRequest {- let (stream, _) = timeout(Duration::from_secs(5), listener.accept())+ accept_http_request_with_timeout(listener, Duration::from_secs(5)).await+}++async fn accept_http_request_with_timeout(+ listener: &TcpListener,+ accept_timeout: Duration,+) -> CapturedHttpRequest {+ let (stream, _) = timeout(accept_timeout, listener.accept()).await.expect("HTTP request should arrive in time").expect("listener accept should succeed");@@ -3043,7 +3063,14 @@ async fn respond_with_status_and_headers(async fn accept_remote_control_backend_connection(listener: &TcpListener,) -> (CapturedWebSocketRequest, WebSocketStream<TcpStream>) {- let (stream, _) = timeout(Duration::from_secs(5), listener.accept())+ accept_remote_control_backend_connection_with_timeout(listener, Duration::from_secs(5)).await+}++async fn accept_remote_control_backend_connection_with_timeout(+ listener: &TcpListener,+ accept_timeout: Duration,+) -> (CapturedWebSocketRequest, WebSocketStream<TcpStream>) {+ let (stream, _) = timeout(accept_timeout, listener.accept()).await.expect("websocket request should arrive in time").expect("listener accept should succeed");diff --git a/codex-rs/app-server-transport/src/transport/remote_control/tests/retry_tests.rs b/codex-rs/app-server-transport/src/transport/remote_control/tests/retry_tests.rsindex 8871f98c6b..cd262f56fa 100644--- a/codex-rs/app-server-transport/src/transport/remote_control/tests/retry_tests.rs+++ b/codex-rs/app-server-transport/src/transport/remote_control/tests/retry_tests.rs@@ -1,4 +1,4 @@-//! Exercises overload retry deadlines over HTTP and WebSocket connections.+//! Exercises overload deadlines and automatic reconnect backoff over real connections.//! Auth reloads must respect the deadline; shutdown must remain prompt.use super::*;@@ -201,3 +201,147 @@ async fn assert_overload_retry_after(status: &str, reject_enrollment: bool) {.expect("shutdown must interrupt the server retry delay").expect("remote task should finish");}++#[tokio::test]+async fn reconnect_backoff_covers_refresh_and_resets_after_healthy_connection() {+ let listener = TcpListener::bind("127.0.0.1:0")+ .await+ .expect("listener should bind");+ let codex_home = TempDir::new().expect("temp dir should create");+ let (transport_event_tx, _transport_event_rx) = mpsc::channel(CHANNEL_CAPACITY);+ let shutdown_token = CancellationToken::new();+ let (remote_task, remote_handle) = start_remote_control(+ RemoteControlStartConfig {+ remote_control_url: remote_control_url_for_listener(&listener),+ installation_id: TEST_INSTALLATION_ID.to_string(),+ policy: RemoteControlPolicy::Allowed,+ },+ Some(remote_control_state_runtime(&codex_home).await),+ remote_control_auth_manager(),+ transport_event_tx,+ shutdown_token.clone(),+ /*app_server_client_name_rx*/ None,+ RemoteControlStartupMode::EnabledEphemeral,+ )+ .await+ .expect("remote control should start");+ let mut status_rx = remote_handle.status_receiver();+ let enroll_request = accept_http_request(&listener).await;+ respond_with_json(+ enroll_request.stream,+ remote_control_server_token_response(+ "srv_e_test",+ "env_test",+ TEST_REMOTE_CONTROL_SERVER_TOKEN,+ ),+ )+ .await;+ let mut websocket = accept_remote_control_connection(&listener).await;+ status_rx+ .wait_for(|status| status.status == RemoteControlConnectionStatus::Connected)+ .await+ .expect("initial connection should succeed");++ // Make the next connection require token work before its handshake.+ remote_handle+ .inner+ .session()+ .current_enrollment+ .lock()+ .await+ .as_mut()+ .expect("enrollment should exist")+ .clear_server_token();+ let disconnected_at = tokio::time::Instant::now();+ websocket.close(None).await.expect("websocket should close");+ drop(websocket);+ let refresh_request =+ accept_http_request_with_timeout(&listener, Duration::from_secs(45)).await;+ assert_eq!(+ refresh_request.request_line,+ "POST /backend-api/wham/remote/control/server/refresh HTTP/1.1"+ );+ assert!(+ disconnected_at.elapsed() >= Duration::from_millis(2_500),+ "the first automatic reconnect must wait before refreshing its token"+ );+ respond_with_json(+ refresh_request.stream,+ remote_control_server_token_response(+ "srv_e_test",+ "env_test",+ TEST_REFRESHED_REMOTE_CONTROL_SERVER_TOKEN,+ ),+ )+ .await;+ let mut websocket = accept_remote_control_connection(&listener).await;+ status_rx+ .wait_for(|status| status.status == RemoteControlConnectionStatus::Connected)+ .await+ .expect("reconnect should succeed");++ let disconnected_at = tokio::time::Instant::now();+ websocket.close(None).await.expect("websocket should close");+ drop(websocket);+ let mut websocket =+ accept_remote_control_connection_with_timeout(&listener, Duration::from_secs(45)).await;+ assert!(+ disconnected_at.elapsed() >= Duration::from_secs(5),+ "a short-lived connection must retain the increased backoff"+ );+ status_rx+ .wait_for(|status| status.status == RemoteControlConnectionStatus::Connected)+ .await+ .expect("reconnect should succeed");++ // Skip most of the healthy interval without crossing the 60-second pong timeout.+ tokio::time::pause();+ tokio::time::advance(Duration::from_secs(45)).await;+ tokio::time::resume();+ // Drive the socket so tungstenite answers pings while we cross the reset threshold.+ tokio::select! {+ _ = tokio::time::sleep(Duration::from_secs(16)) => {}+ _ = async {+ while let Some(frame) = websocket.next().await {+ frame.expect("healthy connection should remain readable");+ }+ } => panic!("healthy connection should remain open"),+ }+ websocket.close(None).await.expect("websocket should close");+ drop(websocket);+ let mut websocket =+ accept_remote_control_connection_with_timeout(&listener, Duration::from_secs(8)).await;+ status_rx+ .wait_for(|status| status.status == RemoteControlConnectionStatus::Connected)+ .await+ .expect("reconnect should succeed");++ websocket.close(None).await.expect("websocket should close");+ drop(websocket);+ status_rx+ .wait_for(|status| status.status == RemoteControlConnectionStatus::Connecting)+ .await+ .expect("closed socket should enter reconnect backoff");+ remote_handle.disable_ephemeral().await;+ // Re-enable immediately, before the websocket task can observe the disabled state.+ remote_handle+ .enable_ephemeral()+ .expect("remote control should enable again");+ let mut websocket =+ accept_remote_control_connection_with_timeout(&listener, Duration::from_secs(2)).await;+ status_rx+ .wait_for(|status| status.status == RemoteControlConnectionStatus::Connected)+ .await+ .expect("explicit connection should succeed");+ websocket.close(None).await.expect("websocket should close");+ drop(websocket);+ status_rx+ .wait_for(|status| status.status == RemoteControlConnectionStatus::Connecting)+ .await+ .expect("closed socket should enter reconnect backoff");+ shutdown_token.cancel();+ timeout(Duration::from_secs(1), remote_task)+ .await+ .expect("shutdown must interrupt reconnect backoff")+ .expect("remote task should finish");+}diff --git a/codex-rs/app-server-transport/src/transport/remote_control/websocket.rs b/codex-rs/app-server-transport/src/transport/remote_control/websocket.rsindex 74a80e3980..7bede8dc65 100644--- a/codex-rs/app-server-transport/src/transport/remote_control/websocket.rs+++ b/codex-rs/app-server-transport/src/transport/remote_control/websocket.rs@@ -39,13 +39,13 @@ use axum::http::HeaderValue;use base64::Engine;use codex_app_server_protocol::RemoteControlConnectionStatus;use codex_app_server_protocol::RemoteControlStatusChangedNotification;-use codex_core::util::backoff;use codex_state::StateRuntime;use codex_utils_rustls_provider::ensure_rustls_crypto_provider;use futures::SinkExt;use futures::StreamExt;use futures::stream::SplitSink;use futures::stream::SplitStream;+use rand::Rng;use std::collections::HashMap;use std::collections::VecDeque;use std::io;@@ -75,6 +75,10 @@ const REMOTE_CONTROL_WEBSOCKET_PONG_TIMEOUT: std::time::Duration =std::time::Duration::from_secs(60);const REMOTE_CONTROL_ACCOUNT_ID_RETRY_INTERVAL: std::time::Duration =std::time::Duration::from_secs(1);+const REMOTE_CONTROL_RECONNECT_BACKOFF_INITIAL: std::time::Duration =+ std::time::Duration::from_secs(5);+const REMOTE_CONTROL_RECONNECT_BACKOFF_RESET_AFTER: std::time::Duration =+ std::time::Duration::from_secs(60);const REMOTE_CONTROL_RECONNECT_BACKOFF_CAP: std::time::Duration =std::time::Duration::from_secs(30);const REMOTE_CONTROL_WEBSOCKET_CONNECT_TIMEOUT: std::time::Duration =@@ -535,6 +539,31 @@ impl RemoteControlWebsocket {desired_state = ?*self.desired_state_rx.borrow(),"app-server remote control websocket connection cycle ended");+ if matches!(+ connection_end_reason,+ ConnectionEndReason::ConnectionWorkerStopped+ ) {+ self.status_publisher+ .publish_status(RemoteControlConnectionStatus::Connecting);+ // Spread automatic reconnects before token refresh or enrollment, too.+ let reconnect_delay = next_reconnect_delay(&mut self.reconnect_attempt);+ tokio::select! {+ biased;+ _ = self.shutdown_token.cancelled() => break,+ _ = auth_owner.invalidated() => break,+ // A disable/enable pair must interrupt the delay even if the latest state is enabled.+ changed = self.desired_state_rx.changed() => {+ if changed.is_err() {+ break;+ }+ if !self.desired_state_rx.borrow().is_enabled() {+ self.status_publisher+ .publish_status(RemoteControlConnectionStatus::Disabled);+ }+ }+ _ = tokio::time::sleep(reconnect_delay) => {}+ }+ }}self.client_tracker.lock().await.shutdown().await;@@ -751,7 +780,6 @@ impl RemoteControlWebsocket {if !self.desired_state_rx.borrow().is_enabled() {return ConnectOutcome::Disabled;}- self.reconnect_attempt = 0;self.auth_recovery = self.auth_manager.unauthorized_recovery();self.status_publisher.publish_status(RemoteControlConnectionStatus::Connected);@@ -838,13 +866,14 @@ impl RemoteControlWebsocket {}async fn run_connection(- &self,+ &mut self,websocket_connection: codex_websocket_client::WebSocketConnection,shutdown_token: CancellationToken,) -> ConnectionEndReason {if !self.auth_manager.owner.is_current() {return ConnectionEndReason::AuthOwnerChanged;}+ let connected_at = tokio::time::Instant::now();self.client_tracker.lock().await.auth = Some(self.auth_manager.owner.clone());let (websocket_writer, websocket_reader) = websocket_connection.split();let mut join_set = tokio::task::JoinSet::new();@@ -881,6 +910,9 @@ impl RemoteControlWebsocket {}_ = join_set.join_next() => ConnectionEndReason::ConnectionWorkerStopped,};+ if connected_at.elapsed() >= REMOTE_CONTROL_RECONNECT_BACKOFF_RESET_AFTER {+ self.reconnect_attempt = 0;+ }shutdown_token.cancel();Self::join_connection_workers(&mut join_set, REMOTE_CONTROL_CONNECTION_SHUTDOWN_TIMEOUT)@@ -1332,14 +1364,13 @@ async fn wait_for_auth_change(}fn next_reconnect_delay(reconnect_attempt: &mut u64) -> std::time::Duration {- let reconnect_delay = backoff(*reconnect_attempt).min(REMOTE_CONTROL_RECONNECT_BACKOFF_CAP);- // Stay at the cap during sustained failures, including duplicate-presence- // conflicts. Success or an auth change resets backoff in the connect loop.- // Stop advancing the exponent too, so a long failure streak cannot overflow it.- if reconnect_delay < REMOTE_CONTROL_RECONNECT_BACKOFF_CAP {- *reconnect_attempt = (*reconnect_attempt).saturating_add(1);- }- reconnect_delay+ let exponent = u32::try_from(*reconnect_attempt).unwrap_or(u32::MAX);+ let backoff = REMOTE_CONTROL_RECONNECT_BACKOFF_INITIAL+ .saturating_mul(2_u32.saturating_pow(exponent))+ .min(REMOTE_CONTROL_RECONNECT_BACKOFF_CAP);+ *reconnect_attempt = reconnect_attempt.saturating_add(1);+ // Keep jitter at the cap, with a lower bound to avoid immediate retries.+ backoff.mul_f64(rand::rng().random_range(0.5..=1.0))}pub(super) async fn connect_remote_control_websocket(diff --git a/codex-rs/app-server-transport/src/transport/remote_control/websocket_retry_tests.rs b/codex-rs/app-server-transport/src/transport/remote_control/websocket_retry_tests.rsindex f5407427a1..1a3e673acb 100644--- a/codex-rs/app-server-transport/src/transport/remote_control/websocket_retry_tests.rs+++ b/codex-rs/app-server-transport/src/transport/remote_control/websocket_retry_tests.rs@@ -81,7 +81,7 @@ async fn assert_conflict_recovery(subscribe_cursor: Option<&str>) {.unwrap();if let Some(rejected_at) = rejected_at {assert!(- Instant::now().duration_since(rejected_at) >= Duration::from_secs(30),+ Instant::now().duration_since(rejected_at) >= Duration::from_secs(15),"sustained HTTP 409 responses must not restart fast retries");}@@ -123,14 +123,17 @@ async fn assert_conflict_recovery(subscribe_cursor: Option<&str>) {// The real loop is sleeping after the second rejection. Skip the// middle of this wait, retaining its boundary assertion below.tokio::time::pause();- tokio::time::advance(Duration::from_secs(28)).await;+ tokio::time::advance(Duration::from_secs(13)).await;tokio::time::resume();}} else {let _server_connection = result.expect("available ownership should allow recovery");- let (websocket, outcome) = connect_task.await.unwrap();+ let (mut websocket, outcome) = connect_task.await.unwrap();assert!(matches!(outcome, ConnectOutcome::Connected(_)));- assert_eq!(websocket.reconnect_attempt, 0);+ assert!(+ next_reconnect_delay(&mut websocket.reconnect_attempt) >= Duration::from_secs(15),+ "a successful handshake must not reset backoff before the connection stays healthy"+ );assert_eq!(current_enrollment.snapshot(), Some(enrollment));assert_eq!(websocket.state.lock().await.subscribe_cursor.as_deref(),@@ -144,14 +147,16 @@ async fn assert_conflict_recovery(subscribe_cursor: Option<&str>) {#[test]fn reconnect_backoff_stays_capped_during_sustained_failures() {let mut reconnect_attempt = 0;- // Allow the initial exponential ramp, then exercise a long failure streak.- for _ in 0..10 {- assert!(next_reconnect_delay(&mut reconnect_attempt) <= Duration::from_secs(30));+ for max_delay in [5, 10, 20] {+ let delay = next_reconnect_delay(&mut reconnect_attempt);+ assert!(delay >= Duration::from_secs(max_delay) / 2);+ assert!(delay <= Duration::from_secs(max_delay));}- for _ in 0..1000 {- assert_eq!(- next_reconnect_delay(&mut reconnect_attempt),- Duration::from_secs(30)- );+ for mut reconnect_attempt in [reconnect_attempt, 9, u64::MAX] {+ for _ in 0..1000 {+ let delay = next_reconnect_delay(&mut reconnect_attempt);+ assert!(delay >= Duration::from_secs(15));+ assert!(delay <= Duration::from_secs(30));+ }}}