Skip to content

armgit / codex

Public Imported from GitHub
0

Lightweight coding agent that runs in your terminal

Rust

Back off automatic remote control reconnects with jitter (#50348)

cooper-oai9d2b60303e83
diff --git a/codex-rs/app-server-transport/src/transport/remote_control/tests.rs b/codex-rs/app-server-transport/src/transport/remote_control/tests.rs
index 2337b2a658..611d540583 100644
--- a/codex-rs/app-server-transport/src/transport/remote_control/tests.rs
+++ b/codex-rs/app-server-transport/src/transport/remote_control/tests.rs
@@ -1136,7 +1136,8 @@ async fn remote_control_transport_reconnects_after_disconnect() {
drop(first_websocket);
let (second_handshake_request, mut second_websocket) =
- accept_remote_control_backend_connection(&listener).await;
+ accept_remote_control_backend_connection_with_timeout(&listener, Duration::from_secs(45))
+ .await;
assert_eq!(
second_handshake_request.headers.get("authorization"),
Some(&format!("Bearer {TEST_REMOTE_CONTROL_SERVER_TOKEN}"))
@@ -1237,7 +1238,8 @@ async fn remote_control_transport_refreshes_server_token_after_websocket_unautho
);
respond_with_status(websocket_request.stream, "401 Unauthorized", "").await;
- let refresh_request = accept_http_request(&listener).await;
+ let refresh_request =
+ accept_http_request_with_timeout(&listener, Duration::from_secs(45)).await;
assert_eq!(
refresh_request.request_line,
"POST /backend-api/wham/remote/control/server/refresh HTTP/1.1"
@@ -1682,7 +1684,8 @@ async fn remote_control_transport_clears_outgoing_buffer_when_backend_acks() {
.expect("first websocket should close");
drop(first_websocket);
- let mut second_websocket = accept_remote_control_connection(&listener).await;
+ let mut second_websocket =
+ accept_remote_control_connection_with_timeout(&listener, Duration::from_secs(45)).await;
send_client_event(
&mut second_websocket,
ClientEnvelope {
@@ -2713,7 +2716,8 @@ async fn remote_control_http_mode_preserves_stale_enrollment_when_reenrollment_f
);
respond_with_status(enroll_request.stream, "500 Internal Server Error", "failed").await;
- let retry_refresh_request = accept_http_request(&listener).await;
+ let retry_refresh_request =
+ accept_http_request_with_timeout(&listener, Duration::from_secs(45)).await;
assert_eq!(
retry_refresh_request.request_line,
"POST /backend-api/wham/remote/control/server/refresh HTTP/1.1"
@@ -2880,7 +2884,9 @@ async fn remote_control_http_mode_preserves_enrollment_after_generic_websocket_4
Some(stale_enrollment.clone())
);
- let (handshake_request, _websocket) = accept_remote_control_backend_connection(&listener).await;
+ let (handshake_request, _websocket) =
+ accept_remote_control_backend_connection_with_timeout(&listener, Duration::from_secs(45))
+ .await;
assert_eq!(
handshake_request.headers.get("x-codex-server-id"),
Some(&stale_enrollment.server_id)
@@ -2946,7 +2952,14 @@ struct CapturedWebSocketRequest {
}
async fn accept_remote_control_connection(listener: &TcpListener) -> WebSocketStream<TcpStream> {
- let (stream, _) = timeout(Duration::from_secs(5), listener.accept())
+ accept_remote_control_connection_with_timeout(listener, Duration::from_secs(5)).await
+}
+
+async fn accept_remote_control_connection_with_timeout(
+ listener: &TcpListener,
+ accept_timeout: Duration,
+) -> WebSocketStream<TcpStream> {
+ let (stream, _) = timeout(accept_timeout, listener.accept())
.await
.expect("remote control should connect in time")
.expect("listener accept should succeed");
@@ -2956,7 +2969,14 @@ async fn accept_remote_control_connection(listener: &TcpListener) -> WebSocketSt
}
async fn accept_http_request(listener: &TcpListener) -> CapturedHttpRequest {
- let (stream, _) = timeout(Duration::from_secs(5), listener.accept())
+ accept_http_request_with_timeout(listener, Duration::from_secs(5)).await
+}
+
+async fn accept_http_request_with_timeout(
+ listener: &TcpListener,
+ accept_timeout: Duration,
+) -> CapturedHttpRequest {
+ let (stream, _) = timeout(accept_timeout, listener.accept())
.await
.expect("HTTP request should arrive in time")
.expect("listener accept should succeed");
@@ -3043,7 +3063,14 @@ async fn respond_with_status_and_headers(
async fn accept_remote_control_backend_connection(
listener: &TcpListener,
) -> (CapturedWebSocketRequest, WebSocketStream<TcpStream>) {
- let (stream, _) = timeout(Duration::from_secs(5), listener.accept())
+ accept_remote_control_backend_connection_with_timeout(listener, Duration::from_secs(5)).await
+}
+
+async fn accept_remote_control_backend_connection_with_timeout(
+ listener: &TcpListener,
+ accept_timeout: Duration,
+) -> (CapturedWebSocketRequest, WebSocketStream<TcpStream>) {
+ let (stream, _) = timeout(accept_timeout, listener.accept())
.await
.expect("websocket request should arrive in time")
.expect("listener accept should succeed");
diff --git a/codex-rs/app-server-transport/src/transport/remote_control/tests/retry_tests.rs b/codex-rs/app-server-transport/src/transport/remote_control/tests/retry_tests.rs
index 8871f98c6b..cd262f56fa 100644
--- a/codex-rs/app-server-transport/src/transport/remote_control/tests/retry_tests.rs
+++ b/codex-rs/app-server-transport/src/transport/remote_control/tests/retry_tests.rs
@@ -1,4 +1,4 @@
-//! Exercises overload retry deadlines over HTTP and WebSocket connections.
+//! Exercises overload deadlines and automatic reconnect backoff over real connections.
//! Auth reloads must respect the deadline; shutdown must remain prompt.
use super::*;
@@ -201,3 +201,147 @@ async fn assert_overload_retry_after(status: &str, reject_enrollment: bool) {
.expect("shutdown must interrupt the server retry delay")
.expect("remote task should finish");
}
+
+#[tokio::test]
+async fn reconnect_backoff_covers_refresh_and_resets_after_healthy_connection() {
+ let listener = TcpListener::bind("127.0.0.1:0")
+ .await
+ .expect("listener should bind");
+ let codex_home = TempDir::new().expect("temp dir should create");
+ let (transport_event_tx, _transport_event_rx) = mpsc::channel(CHANNEL_CAPACITY);
+ let shutdown_token = CancellationToken::new();
+ let (remote_task, remote_handle) = start_remote_control(
+ RemoteControlStartConfig {
+ remote_control_url: remote_control_url_for_listener(&listener),
+ installation_id: TEST_INSTALLATION_ID.to_string(),
+ policy: RemoteControlPolicy::Allowed,
+ },
+ Some(remote_control_state_runtime(&codex_home).await),
+ remote_control_auth_manager(),
+ transport_event_tx,
+ shutdown_token.clone(),
+ /*app_server_client_name_rx*/ None,
+ RemoteControlStartupMode::EnabledEphemeral,
+ )
+ .await
+ .expect("remote control should start");
+ let mut status_rx = remote_handle.status_receiver();
+ let enroll_request = accept_http_request(&listener).await;
+ respond_with_json(
+ enroll_request.stream,
+ remote_control_server_token_response(
+ "srv_e_test",
+ "env_test",
+ TEST_REMOTE_CONTROL_SERVER_TOKEN,
+ ),
+ )
+ .await;
+ let mut websocket = accept_remote_control_connection(&listener).await;
+ status_rx
+ .wait_for(|status| status.status == RemoteControlConnectionStatus::Connected)
+ .await
+ .expect("initial connection should succeed");
+
+ // Make the next connection require token work before its handshake.
+ remote_handle
+ .inner
+ .session()
+ .current_enrollment
+ .lock()
+ .await
+ .as_mut()
+ .expect("enrollment should exist")
+ .clear_server_token();
+ let disconnected_at = tokio::time::Instant::now();
+ websocket.close(None).await.expect("websocket should close");
+ drop(websocket);
+ let refresh_request =
+ accept_http_request_with_timeout(&listener, Duration::from_secs(45)).await;
+ assert_eq!(
+ refresh_request.request_line,
+ "POST /backend-api/wham/remote/control/server/refresh HTTP/1.1"
+ );
+ assert!(
+ disconnected_at.elapsed() >= Duration::from_millis(2_500),
+ "the first automatic reconnect must wait before refreshing its token"
+ );
+ respond_with_json(
+ refresh_request.stream,
+ remote_control_server_token_response(
+ "srv_e_test",
+ "env_test",
+ TEST_REFRESHED_REMOTE_CONTROL_SERVER_TOKEN,
+ ),
+ )
+ .await;
+ let mut websocket = accept_remote_control_connection(&listener).await;
+ status_rx
+ .wait_for(|status| status.status == RemoteControlConnectionStatus::Connected)
+ .await
+ .expect("reconnect should succeed");
+
+ let disconnected_at = tokio::time::Instant::now();
+ websocket.close(None).await.expect("websocket should close");
+ drop(websocket);
+ let mut websocket =
+ accept_remote_control_connection_with_timeout(&listener, Duration::from_secs(45)).await;
+ assert!(
+ disconnected_at.elapsed() >= Duration::from_secs(5),
+ "a short-lived connection must retain the increased backoff"
+ );
+ status_rx
+ .wait_for(|status| status.status == RemoteControlConnectionStatus::Connected)
+ .await
+ .expect("reconnect should succeed");
+
+ // Skip most of the healthy interval without crossing the 60-second pong timeout.
+ tokio::time::pause();
+ tokio::time::advance(Duration::from_secs(45)).await;
+ tokio::time::resume();
+ // Drive the socket so tungstenite answers pings while we cross the reset threshold.
+ tokio::select! {
+ _ = tokio::time::sleep(Duration::from_secs(16)) => {}
+ _ = async {
+ while let Some(frame) = websocket.next().await {
+ frame.expect("healthy connection should remain readable");
+ }
+ } => panic!("healthy connection should remain open"),
+ }
+ websocket.close(None).await.expect("websocket should close");
+ drop(websocket);
+ let mut websocket =
+ accept_remote_control_connection_with_timeout(&listener, Duration::from_secs(8)).await;
+ status_rx
+ .wait_for(|status| status.status == RemoteControlConnectionStatus::Connected)
+ .await
+ .expect("reconnect should succeed");
+
+ websocket.close(None).await.expect("websocket should close");
+ drop(websocket);
+ status_rx
+ .wait_for(|status| status.status == RemoteControlConnectionStatus::Connecting)
+ .await
+ .expect("closed socket should enter reconnect backoff");
+ remote_handle.disable_ephemeral().await;
+ // Re-enable immediately, before the websocket task can observe the disabled state.
+ remote_handle
+ .enable_ephemeral()
+ .expect("remote control should enable again");
+ let mut websocket =
+ accept_remote_control_connection_with_timeout(&listener, Duration::from_secs(2)).await;
+ status_rx
+ .wait_for(|status| status.status == RemoteControlConnectionStatus::Connected)
+ .await
+ .expect("explicit connection should succeed");
+ websocket.close(None).await.expect("websocket should close");
+ drop(websocket);
+ status_rx
+ .wait_for(|status| status.status == RemoteControlConnectionStatus::Connecting)
+ .await
+ .expect("closed socket should enter reconnect backoff");
+ shutdown_token.cancel();
+ timeout(Duration::from_secs(1), remote_task)
+ .await
+ .expect("shutdown must interrupt reconnect backoff")
+ .expect("remote task should finish");
+}
diff --git a/codex-rs/app-server-transport/src/transport/remote_control/websocket.rs b/codex-rs/app-server-transport/src/transport/remote_control/websocket.rs
index 74a80e3980..7bede8dc65 100644
--- a/codex-rs/app-server-transport/src/transport/remote_control/websocket.rs
+++ b/codex-rs/app-server-transport/src/transport/remote_control/websocket.rs
@@ -39,13 +39,13 @@ use axum::http::HeaderValue;
use base64::Engine;
use codex_app_server_protocol::RemoteControlConnectionStatus;
use codex_app_server_protocol::RemoteControlStatusChangedNotification;
-use codex_core::util::backoff;
use codex_state::StateRuntime;
use codex_utils_rustls_provider::ensure_rustls_crypto_provider;
use futures::SinkExt;
use futures::StreamExt;
use futures::stream::SplitSink;
use futures::stream::SplitStream;
+use rand::Rng;
use std::collections::HashMap;
use std::collections::VecDeque;
use std::io;
@@ -75,6 +75,10 @@ const REMOTE_CONTROL_WEBSOCKET_PONG_TIMEOUT: std::time::Duration =
std::time::Duration::from_secs(60);
const REMOTE_CONTROL_ACCOUNT_ID_RETRY_INTERVAL: std::time::Duration =
std::time::Duration::from_secs(1);
+const REMOTE_CONTROL_RECONNECT_BACKOFF_INITIAL: std::time::Duration =
+ std::time::Duration::from_secs(5);
+const REMOTE_CONTROL_RECONNECT_BACKOFF_RESET_AFTER: std::time::Duration =
+ std::time::Duration::from_secs(60);
const REMOTE_CONTROL_RECONNECT_BACKOFF_CAP: std::time::Duration =
std::time::Duration::from_secs(30);
const REMOTE_CONTROL_WEBSOCKET_CONNECT_TIMEOUT: std::time::Duration =
@@ -535,6 +539,31 @@ impl RemoteControlWebsocket {
desired_state = ?*self.desired_state_rx.borrow(),
"app-server remote control websocket connection cycle ended"
);
+ if matches!(
+ connection_end_reason,
+ ConnectionEndReason::ConnectionWorkerStopped
+ ) {
+ self.status_publisher
+ .publish_status(RemoteControlConnectionStatus::Connecting);
+ // Spread automatic reconnects before token refresh or enrollment, too.
+ let reconnect_delay = next_reconnect_delay(&mut self.reconnect_attempt);
+ tokio::select! {
+ biased;
+ _ = self.shutdown_token.cancelled() => break,
+ _ = auth_owner.invalidated() => break,
+ // A disable/enable pair must interrupt the delay even if the latest state is enabled.
+ changed = self.desired_state_rx.changed() => {
+ if changed.is_err() {
+ break;
+ }
+ if !self.desired_state_rx.borrow().is_enabled() {
+ self.status_publisher
+ .publish_status(RemoteControlConnectionStatus::Disabled);
+ }
+ }
+ _ = tokio::time::sleep(reconnect_delay) => {}
+ }
+ }
}
self.client_tracker.lock().await.shutdown().await;
@@ -751,7 +780,6 @@ impl RemoteControlWebsocket {
if !self.desired_state_rx.borrow().is_enabled() {
return ConnectOutcome::Disabled;
}
- self.reconnect_attempt = 0;
self.auth_recovery = self.auth_manager.unauthorized_recovery();
self.status_publisher
.publish_status(RemoteControlConnectionStatus::Connected);
@@ -838,13 +866,14 @@ impl RemoteControlWebsocket {
}
async fn run_connection(
- &self,
+ &mut self,
websocket_connection: codex_websocket_client::WebSocketConnection,
shutdown_token: CancellationToken,
) -> ConnectionEndReason {
if !self.auth_manager.owner.is_current() {
return ConnectionEndReason::AuthOwnerChanged;
}
+ let connected_at = tokio::time::Instant::now();
self.client_tracker.lock().await.auth = Some(self.auth_manager.owner.clone());
let (websocket_writer, websocket_reader) = websocket_connection.split();
let mut join_set = tokio::task::JoinSet::new();
@@ -881,6 +910,9 @@ impl RemoteControlWebsocket {
}
_ = join_set.join_next() => ConnectionEndReason::ConnectionWorkerStopped,
};
+ if connected_at.elapsed() >= REMOTE_CONTROL_RECONNECT_BACKOFF_RESET_AFTER {
+ self.reconnect_attempt = 0;
+ }
shutdown_token.cancel();
Self::join_connection_workers(&mut join_set, REMOTE_CONTROL_CONNECTION_SHUTDOWN_TIMEOUT)
@@ -1332,14 +1364,13 @@ async fn wait_for_auth_change(
}
fn next_reconnect_delay(reconnect_attempt: &mut u64) -> std::time::Duration {
- let reconnect_delay = backoff(*reconnect_attempt).min(REMOTE_CONTROL_RECONNECT_BACKOFF_CAP);
- // Stay at the cap during sustained failures, including duplicate-presence
- // conflicts. Success or an auth change resets backoff in the connect loop.
- // Stop advancing the exponent too, so a long failure streak cannot overflow it.
- if reconnect_delay < REMOTE_CONTROL_RECONNECT_BACKOFF_CAP {
- *reconnect_attempt = (*reconnect_attempt).saturating_add(1);
- }
- reconnect_delay
+ let exponent = u32::try_from(*reconnect_attempt).unwrap_or(u32::MAX);
+ let backoff = REMOTE_CONTROL_RECONNECT_BACKOFF_INITIAL
+ .saturating_mul(2_u32.saturating_pow(exponent))
+ .min(REMOTE_CONTROL_RECONNECT_BACKOFF_CAP);
+ *reconnect_attempt = reconnect_attempt.saturating_add(1);
+ // Keep jitter at the cap, with a lower bound to avoid immediate retries.
+ backoff.mul_f64(rand::rng().random_range(0.5..=1.0))
}
pub(super) async fn connect_remote_control_websocket(
diff --git a/codex-rs/app-server-transport/src/transport/remote_control/websocket_retry_tests.rs b/codex-rs/app-server-transport/src/transport/remote_control/websocket_retry_tests.rs
index f5407427a1..1a3e673acb 100644
--- a/codex-rs/app-server-transport/src/transport/remote_control/websocket_retry_tests.rs
+++ b/codex-rs/app-server-transport/src/transport/remote_control/websocket_retry_tests.rs
@@ -81,7 +81,7 @@ async fn assert_conflict_recovery(subscribe_cursor: Option<&str>) {
.unwrap();
if let Some(rejected_at) = rejected_at {
assert!(
- Instant::now().duration_since(rejected_at) >= Duration::from_secs(30),
+ Instant::now().duration_since(rejected_at) >= Duration::from_secs(15),
"sustained HTTP 409 responses must not restart fast retries"
);
}
@@ -123,14 +123,17 @@ async fn assert_conflict_recovery(subscribe_cursor: Option<&str>) {
// The real loop is sleeping after the second rejection. Skip the
// middle of this wait, retaining its boundary assertion below.
tokio::time::pause();
- tokio::time::advance(Duration::from_secs(28)).await;
+ tokio::time::advance(Duration::from_secs(13)).await;
tokio::time::resume();
}
} else {
let _server_connection = result.expect("available ownership should allow recovery");
- let (websocket, outcome) = connect_task.await.unwrap();
+ let (mut websocket, outcome) = connect_task.await.unwrap();
assert!(matches!(outcome, ConnectOutcome::Connected(_)));
- assert_eq!(websocket.reconnect_attempt, 0);
+ assert!(
+ next_reconnect_delay(&mut websocket.reconnect_attempt) >= Duration::from_secs(15),
+ "a successful handshake must not reset backoff before the connection stays healthy"
+ );
assert_eq!(current_enrollment.snapshot(), Some(enrollment));
assert_eq!(
websocket.state.lock().await.subscribe_cursor.as_deref(),
@@ -144,14 +147,16 @@ async fn assert_conflict_recovery(subscribe_cursor: Option<&str>) {
#[test]
fn reconnect_backoff_stays_capped_during_sustained_failures() {
let mut reconnect_attempt = 0;
- // Allow the initial exponential ramp, then exercise a long failure streak.
- for _ in 0..10 {
- assert!(next_reconnect_delay(&mut reconnect_attempt) <= Duration::from_secs(30));
+ for max_delay in [5, 10, 20] {
+ let delay = next_reconnect_delay(&mut reconnect_attempt);
+ assert!(delay >= Duration::from_secs(max_delay) / 2);
+ assert!(delay <= Duration::from_secs(max_delay));
}
- for _ in 0..1000 {
- assert_eq!(
- next_reconnect_delay(&mut reconnect_attempt),
- Duration::from_secs(30)
- );
+ for mut reconnect_attempt in [reconnect_attempt, 9, u64::MAX] {
+ for _ in 0..1000 {
+ let delay = next_reconnect_delay(&mut reconnect_attempt);
+ assert!(delay >= Duration::from_secs(15));
+ assert!(delay <= Duration::from_secs(30));
+ }
}
}