Lightweight coding agent that runs in your terminal
Rust
Reset registry retries when refreshing Noise bundles (#39852)
richardopenai2151d3a5b78c
diff --git a/codex-rs/core/tests/suite/remote_env.rs b/codex-rs/core/tests/suite/remote_env.rsindex 3bcd720afd..1fad9556eb 100644--- a/codex-rs/core/tests/suite/remote_env.rs+++ b/codex-rs/core/tests/suite/remote_env.rs@@ -1169,16 +1169,26 @@ impl NoiseRendezvousConnectProvider for FailingNoiseConnectProvider {}}-struct ReadyNoiseConnectProvider {+struct OfflineThenReadyNoiseConnectProvider {websocket_url: String,executor_public_key: NoiseChannelPublicKey,+ calls: AtomicUsize,}-impl NoiseRendezvousConnectProvider for ReadyNoiseConnectProvider {+impl NoiseRendezvousConnectProvider for OfflineThenReadyNoiseConnectProvider {fn connect_bundle(&self,_: NoiseChannelPublicKey,) -> BoxFuture<'_, std::result::Result<NoiseRendezvousConnectBundle, ExecServerError>> {+ if self.calls.fetch_add(1, Ordering::Relaxed) == 0 {+ return Box::pin(async {+ Err(ExecServerError::EnvironmentRegistryHttp {+ status: http::StatusCode::CONFLICT,+ code: Some("environment_offline".to_string()),+ message: "test environment is offline".to_string(),+ })+ });+ }let bundle = NoiseRendezvousConnectBundle {websocket_url: self.websocket_url.clone(),environment_id: REMOTE_ENVIRONMENT_ID.to_string(),@@ -1824,6 +1834,11 @@ async fn ready_before_selection_resolves_resumed_thread_capability_root_after_wa} else {stale_root.clone()};+ let provider = Arc::new(OfflineThenReadyNoiseConnectProvider {+ websocket_url: format!("{rendezvous_url}/relay?role=harness"),+ executor_public_key,+ calls: AtomicUsize::new(0),+ });let environment = test.thread_manager.environment_manager()@@ -1836,10 +1851,7 @@ async fn ready_before_selection_resolves_resumed_thread_capability_root_after_waVec::new()},}),- Arc::new(ReadyNoiseConnectProvider {- websocket_url: format!("{rendezvous_url}/relay?role=harness"),- executor_public_key,- }),+ provider.clone(),)?.context("Ready-first report should create the environment")?;@@ -1914,6 +1926,7 @@ async fn ready_before_selection_resolves_resumed_thread_capability_root_after_walet requests = response_mock.requests();assert_eq!(requests.len(), 2);+ assert_eq!(provider.calls.load(Ordering::Relaxed), 2);// Provisioning was reported ready before selection, but selection materialization remains// nonblocking while the transport starts.// The first request may legally see either Starting or Ready; the wait makes step two ready.diff --git a/codex-rs/exec-server/src/client_transport.rs b/codex-rs/exec-server/src/client_transport.rsindex 318f1adc62..add01f69d4 100644--- a/codex-rs/exec-server/src/client_transport.rs+++ b/codex-rs/exec-server/src/client_transport.rs@@ -314,6 +314,7 @@ impl ExecServerClient {{refreshed_unauthorized_bundle = true;deadline = Instant::now() + INITIAL_REGISTRY_OPERATION_TIMEOUT;+ retries = 0;result = connect_bundle().await;}result => return result,diff --git a/codex-rs/exec-server/src/client_transport_tests.rs b/codex-rs/exec-server/src/client_transport_tests.rsindex 846dab9bc4..411dfc0e8d 100644--- a/codex-rs/exec-server/src/client_transport_tests.rs+++ b/codex-rs/exec-server/src/client_transport_tests.rs@@ -1,4 +1,5 @@use std::collections::VecDeque;+use std::future::Future;use std::sync::Arc;use std::sync::Mutex;@@ -24,6 +25,7 @@ use crate::NoiseRendezvousConnectProvider;use crate::client_api::DEFAULT_REMOTE_EXEC_SERVER_CONNECT_TIMEOUT;use crate::client_api::DEFAULT_REMOTE_EXEC_SERVER_INITIALIZE_TIMEOUT;+#[derive(Default)]struct SequenceNoiseConnectProvider {bundles:Mutex<VecDeque<BoxFuture<'static, Result<NoiseRendezvousConnectBundle, ExecServerError>>>>,@@ -32,17 +34,29 @@ struct SequenceNoiseConnectProvider {}impl SequenceNoiseConnectProvider {- fn new(bundles: Vec<Result<NoiseRendezvousConnectBundle, ExecServerError>>) -> Self {- Self {- bundles: Mutex::new(- bundles- .into_iter()- .map(|bundle| futures::future::ready(bundle).boxed())- .collect(),- ),- returned_urls: Mutex::new(Vec::new()),- requested_keys: Mutex::new(Vec::new()),- }+ fn push_response(+ &self,+ response: impl Future<Output = Result<NoiseRendezvousConnectBundle, ExecServerError>>+ + Send+ + 'static,+ ) {+ self.bundles.lock().unwrap().push_back(response.boxed());+ }++ fn push_error(&self, error: ExecServerError) {+ self.push_response(futures::future::ready(Err(error)));+ }++ fn push_pending(&self) {+ self.push_response(futures::future::pending());+ }++ fn requested_keys(&self) -> Vec<NoiseChannelPublicKey> {+ self.requested_keys.lock().unwrap().clone()+ }++ fn assert_requested_identity(&self, identity: &NoiseChannelIdentity, requests: usize) {+ assert_eq!(self.requested_keys(), vec![identity.public_key(); requests]);}fn returned_urls(&self) -> Vec<String> {@@ -51,6 +65,27 @@ impl SequenceNoiseConnectProvider {.unwrap_or_else(std::sync::PoisonError::into_inner).clone()}++ async fn connect(+ self: &Arc<Self>,+ identity: &NoiseChannelIdentity,+ ) -> Result<+ (+ super::JsonRpcConnection,+ super::ExecServerClientConnectOptions,+ ),+ ExecServerError,+ > {+ let provider: Arc<dyn NoiseRendezvousConnectProvider> = self.clone();+ ExecServerClient::open_initial_noise_rendezvous_connection(+ &provider,+ identity,+ codex_http_client::HttpClientFactory::new(+ codex_http_client::OutboundProxyPolicy::ReqwestDefault,+ ),+ )+ .await+ }}impl NoiseRendezvousConnectProvider for SequenceNoiseConnectProvider {@@ -98,147 +133,97 @@ fn registry_error(status: http::StatusCode, code: &str) -> ExecServerError {#[tokio::test(start_paused = true)]async fn initial_noise_connection_bounds_offline_retries() -> Result<()> {- let sequence = Arc::new(SequenceNoiseConnectProvider::new(- (0..=INITIAL_REGISTRY_MAX_RETRIES)- .map(|_| {- Err(registry_error(- http::StatusCode::CONFLICT,- "environment_offline",- ))- })- .collect(),- ));- let provider: Arc<dyn NoiseRendezvousConnectProvider> = sequence.clone();+ let sequence = Arc::new(SequenceNoiseConnectProvider::default());+ for _ in 0..=INITIAL_REGISTRY_MAX_RETRIES {+ sequence.push_error(registry_error(+ http::StatusCode::CONFLICT,+ "environment_offline",+ ));+ }let identity = NoiseChannelIdentity::generate()?;let started = tokio::time::Instant::now();- let error = ExecServerClient::open_initial_noise_rendezvous_connection(- &provider,- &identity,- codex_http_client::HttpClientFactory::new(- codex_http_client::OutboundProxyPolicy::ReqwestDefault,- ),- )- .await- .err()- .expect("offline retries must end");+ let error = sequence+ .connect(&identity)+ .await+ .err()+ .expect("offline retries must end");assert!(crate::client::is_environment_offline_error(&error));- let requested_keys = sequence.requested_keys.lock().unwrap();- assert!((4..=INITIAL_REGISTRY_MAX_RETRIES as usize + 1).contains(&requested_keys.len()));- assert_eq!(- *requested_keys,- vec![identity.public_key(); requested_keys.len()]- );+ let requests = sequence.requested_keys().len();+ assert!((4..=INITIAL_REGISTRY_MAX_RETRIES as usize + 1).contains(&requests));+ sequence.assert_requested_identity(&identity, requests);assert!(started.elapsed() <= INITIAL_REGISTRY_OPERATION_TIMEOUT);Ok(())}#[tokio::test(start_paused = true)]async fn initial_noise_connection_bounds_a_stalled_retry_request() -> Result<()> {- let sequence = Arc::new(SequenceNoiseConnectProvider::new(vec![Err(- registry_error(http::StatusCode::CONFLICT, "environment_offline"),- )]));- sequence- .bundles- .lock()- .unwrap()- .extend((0..INITIAL_REGISTRY_MAX_RETRIES).map(|_| {- futures::future::pending::<Result<NoiseRendezvousConnectBundle, ExecServerError>>()- .boxed()- }));- let provider: Arc<dyn NoiseRendezvousConnectProvider> = sequence.clone();+ let sequence = Arc::new(SequenceNoiseConnectProvider::default());+ sequence.push_error(registry_error(+ http::StatusCode::CONFLICT,+ "environment_offline",+ ));+ for _ in 0..INITIAL_REGISTRY_MAX_RETRIES {+ sequence.push_pending();+ }let identity = NoiseChannelIdentity::generate()?;let started = tokio::time::Instant::now();- let error = ExecServerClient::open_initial_noise_rendezvous_connection(- &provider,- &identity,- codex_http_client::HttpClientFactory::new(- codex_http_client::OutboundProxyPolicy::ReqwestDefault,- ),- )- .await- .err()- .expect("stalled retry must time out");+ let error = sequence+ .connect(&identity)+ .await+ .err()+ .expect("stalled retry must time out");assert!(matches!(error,ExecServerError::EnvironmentRegistryRequest(error) if error.is_timeout()));assert_eq!(started.elapsed(), INITIAL_REGISTRY_OPERATION_TIMEOUT);- let requested_keys = sequence.requested_keys.lock().unwrap();- assert!((2..=3).contains(&requested_keys.len()));- assert_eq!(- *requested_keys,- vec![identity.public_key(); requested_keys.len()]- );+ let requests = sequence.requested_keys().len();+ assert!((2..=3).contains(&requests));+ sequence.assert_requested_identity(&identity, requests);Ok(())}#[tokio::test(start_paused = true)]async fn initial_noise_connection_bounds_a_stalled_initial_request() -> Result<()> {- let sequence = Arc::new(SequenceNoiseConnectProvider::new(vec![]));- sequence- .bundles- .lock()- .unwrap()- .extend((0..=INITIAL_REGISTRY_MAX_RETRIES).map(|_| {- futures::future::pending::<Result<NoiseRendezvousConnectBundle, ExecServerError>>()- .boxed()- }));- let provider: Arc<dyn NoiseRendezvousConnectProvider> = sequence.clone();+ let sequence = Arc::new(SequenceNoiseConnectProvider::default());+ for _ in 0..=INITIAL_REGISTRY_MAX_RETRIES {+ sequence.push_pending();+ }let identity = NoiseChannelIdentity::generate()?;let started = tokio::time::Instant::now();- let error = ExecServerClient::open_initial_noise_rendezvous_connection(- &provider,- &identity,- codex_http_client::HttpClientFactory::new(- codex_http_client::OutboundProxyPolicy::ReqwestDefault,- ),- )- .await- .err()- .expect("stalled initial request must time out");+ let error = sequence+ .connect(&identity)+ .await+ .err()+ .expect("stalled initial request must time out");assert!(matches!(error,ExecServerError::EnvironmentRegistryRequest(error) if error.is_timeout()));assert_eq!(started.elapsed(), INITIAL_REGISTRY_OPERATION_TIMEOUT);- let requested_keys = sequence.requested_keys.lock().unwrap();- assert!((2..=3).contains(&requested_keys.len()));- assert_eq!(- *requested_keys,- vec![identity.public_key(); requested_keys.len()]- );+ let requests = sequence.requested_keys().len();+ assert!((2..=3).contains(&requests));+ sequence.assert_requested_identity(&identity, requests);Ok(())}#[tokio::test(start_paused = true)]async fn initial_noise_connection_retries_a_stalled_initial_request() -> Result<()> {- let sequence = Arc::new(SequenceNoiseConnectProvider::new(vec![]));- sequence.bundles.lock().unwrap().extend([- futures::future::pending::<Result<NoiseRendezvousConnectBundle, ExecServerError>>().boxed(),- futures::future::ready(Err(registry_error(- http::StatusCode::FORBIDDEN,- "forbidden",- )))- .boxed(),- ]);- let provider: Arc<dyn NoiseRendezvousConnectProvider> = sequence.clone();+ let sequence = Arc::new(SequenceNoiseConnectProvider::default());+ sequence.push_pending();+ sequence.push_error(registry_error(http::StatusCode::FORBIDDEN, "forbidden"));let identity = NoiseChannelIdentity::generate()?;let started = tokio::time::Instant::now();- let error = ExecServerClient::open_initial_noise_rendezvous_connection(- &provider,- &identity,- codex_http_client::HttpClientFactory::new(- codex_http_client::OutboundProxyPolicy::ReqwestDefault,- ),- )- .await- .err()- .expect("terminal response must stop the retry sequence");+ let error = sequence+ .connect(&identity)+ .await+ .err()+ .expect("terminal response must stop the retry sequence");assert!(matches!(error,@@ -249,10 +234,7 @@ async fn initial_noise_connection_retries_a_stalled_initial_request() -> Result<));assert!(started.elapsed() >= INITIAL_REGISTRY_REQUEST_TIMEOUT);assert!(started.elapsed() < INITIAL_REGISTRY_OPERATION_TIMEOUT);- assert_eq!(- *sequence.requested_keys.lock().unwrap(),- vec![identity.public_key(); 2]- );+ sequence.assert_requested_identity(&identity, /*requests*/ 2);Ok(())}@@ -265,23 +247,16 @@ async fn initial_noise_connection_retries_transient_registry_statuses() -> Resulhttp::StatusCode::BAD_GATEWAY,http::StatusCode::SERVICE_UNAVAILABLE,] {- let sequence = Arc::new(SequenceNoiseConnectProvider::new(vec![- Err(registry_error(status, "temporarily_unavailable")),- Err(registry_error(http::StatusCode::FORBIDDEN, "forbidden")),- ]));- let provider: Arc<dyn NoiseRendezvousConnectProvider> = sequence.clone();+ let sequence = Arc::new(SequenceNoiseConnectProvider::default());+ sequence.push_error(registry_error(status, "temporarily_unavailable"));+ sequence.push_error(registry_error(http::StatusCode::FORBIDDEN, "forbidden"));let identity = NoiseChannelIdentity::generate()?;- let error = ExecServerClient::open_initial_noise_rendezvous_connection(- &provider,- &identity,- codex_http_client::HttpClientFactory::new(- codex_http_client::OutboundProxyPolicy::ReqwestDefault,- ),- )- .await- .err()- .expect("terminal response must stop the retry sequence");+ let error = sequence+ .connect(&identity)+ .await+ .err()+ .expect("terminal response must stop the retry sequence");assert!(matches!(error,@@ -290,36 +265,25 @@ async fn initial_noise_connection_retries_transient_registry_statuses() -> Resul..}));- assert_eq!(- *sequence.requested_keys.lock().unwrap(),- vec![identity.public_key(); 2],- "registry status {status} should be retried"- );+ sequence.assert_requested_identity(&identity, /*requests*/ 2);}Ok(())}#[tokio::test(start_paused = true)]async fn initial_noise_connection_retries_registry_request_timeouts() -> Result<()> {- let sequence = Arc::new(SequenceNoiseConnectProvider::new(vec![- Err(ExecServerError::EnvironmentRegistryRequest(- codex_http_client::RouteAwareRequestError::Timeout,- )),- Err(registry_error(http::StatusCode::FORBIDDEN, "forbidden")),- ]));- let provider: Arc<dyn NoiseRendezvousConnectProvider> = sequence.clone();+ let sequence = Arc::new(SequenceNoiseConnectProvider::default());+ sequence.push_error(ExecServerError::EnvironmentRegistryRequest(+ codex_http_client::RouteAwareRequestError::Timeout,+ ));+ sequence.push_error(registry_error(http::StatusCode::FORBIDDEN, "forbidden"));let identity = NoiseChannelIdentity::generate()?;- let error = ExecServerClient::open_initial_noise_rendezvous_connection(- &provider,- &identity,- codex_http_client::HttpClientFactory::new(- codex_http_client::OutboundProxyPolicy::ReqwestDefault,- ),- )- .await- .err()- .expect("terminal response must stop the retry sequence");+ let error = sequence+ .connect(&identity)+ .await+ .err()+ .expect("terminal response must stop the retry sequence");assert!(matches!(error,@@ -328,10 +292,7 @@ async fn initial_noise_connection_retries_registry_request_timeouts() -> Result<..}));- assert_eq!(- *sequence.requested_keys.lock().unwrap(),- vec![identity.public_key(); 2]- );+ sequence.assert_requested_identity(&identity, /*requests*/ 2);Ok(())}@@ -347,34 +308,24 @@ async fn initial_noise_connection_does_not_retry_permanent_registry_errors() ->] {// A terminal error must also stop a retry sequence already in progress.for initial_offline in [false, true] {- let mut responses = Vec::new();+ let sequence = Arc::new(SequenceNoiseConnectProvider::default());if initial_offline {- responses.push(Err(registry_error(+ sequence.push_error(registry_error(http::StatusCode::CONFLICT,"environment_offline",- )));+ ));}- responses.push(Err(registry_error(status, code)));- let sequence = Arc::new(SequenceNoiseConnectProvider::new(responses));- let provider: Arc<dyn NoiseRendezvousConnectProvider> = sequence.clone();+ sequence.push_error(registry_error(status, code));let identity = NoiseChannelIdentity::generate()?;- let error = ExecServerClient::open_initial_noise_rendezvous_connection(- &provider,- &identity,- codex_http_client::HttpClientFactory::new(- codex_http_client::OutboundProxyPolicy::ReqwestDefault,- ),- )- .await- .err()- .expect("other errors must propagate");+ let error = sequence+ .connect(&identity)+ .await+ .err()+ .expect("other errors must propagate");assert!(matches!(error, ExecServerError::EnvironmentRegistryHttp { status: actual_status, code: Some(actual_code), .. } if actual_status == status && actual_code == code));- assert_eq!(- sequence.requested_keys.lock().unwrap().len(),- 1 + usize::from(initial_offline)- );+ sequence.assert_requested_identity(&identity, 1 + usize::from(initial_offline));}}Ok(())@@ -382,9 +333,11 @@ async fn initial_noise_connection_does_not_retry_permanent_registry_errors() ->#[tokio::test(start_paused = true)]async fn noise_session_resume_leaves_offline_retries_to_recovery() -> Result<()> {- let sequence = Arc::new(SequenceNoiseConnectProvider::new(vec![Err(- registry_error(http::StatusCode::CONFLICT, "environment_offline"),- )]));+ let sequence = Arc::new(SequenceNoiseConnectProvider::default());+ sequence.push_error(registry_error(+ http::StatusCode::CONFLICT,+ "environment_offline",+ ));let identity = NoiseChannelIdentity::generate()?;let strategy = ExecServerReconnectStrategy::NoiseRendezvous {provider: sequence.clone(),@@ -404,15 +357,13 @@ async fn noise_session_resume_leaves_offline_retries_to_recovery() -> Result<()>.expect("resume must return the offline error");assert!(crate::client::is_environment_offline_error(&error));assert_eq!(started.elapsed(), std::time::Duration::ZERO);- assert_eq!(- *sequence.requested_keys.lock().unwrap(),- vec![identity.public_key()]- );+ sequence.assert_requested_identity(&identity, /*requests*/ 1);Ok(())}#[tokio::test]-async fn initial_noise_connection_refreshes_bundle_after_unauthorized_handshake() -> Result<()> {+async fn initial_noise_connection_refreshes_bundle_after_exhausting_initial_retries() -> Result<()>+{let unauthorized_listener = TcpListener::bind("127.0.0.1:0").await?;let unauthorized_url = format!("ws://{}", unauthorized_listener.local_addr()?);let unauthorized_server = tokio::spawn(async move {@@ -434,57 +385,46 @@ async fn initial_noise_connection_refreshes_bundle_after_unauthorized_handshake(let _websocket = accept_async(socket).await?;anyhow::Ok(())});- let sequence = Arc::new(SequenceNoiseConnectProvider::new(vec![]));+ let sequence = Arc::new(SequenceNoiseConnectProvider::default());let unauthorized_bundle = test_bundle(unauthorized_url.clone())?;let accepted_bundle = test_bundle(accepted_url.clone())?;- sequence.bundles.lock().unwrap().extend([- async {- tokio::time::pause();- Err(registry_error(- http::StatusCode::CONFLICT,- "environment_offline",- ))- }- .boxed(),- async move {- tokio::time::resume();- Ok(unauthorized_bundle)- }- .boxed(),- async {- tokio::time::pause();- Err(registry_error(- http::StatusCode::CONFLICT,- "environment_offline",- ))- }- .boxed(),- async move {- tokio::time::resume();- Ok(accepted_bundle)- }- .boxed(),- ]);- let provider: Arc<dyn NoiseRendezvousConnectProvider> = sequence.clone();+ sequence.push_response(async {+ tokio::time::pause();+ Err(registry_error(+ http::StatusCode::CONFLICT,+ "environment_offline",+ ))+ });+ for _ in 1..INITIAL_REGISTRY_MAX_RETRIES {+ sequence.push_error(registry_error(+ http::StatusCode::CONFLICT,+ "environment_offline",+ ));+ }+ sequence.push_response(async move {+ tokio::time::resume();+ Ok(unauthorized_bundle)+ });+ sequence.push_response(async {+ tokio::time::pause();+ Err(registry_error(+ http::StatusCode::CONFLICT,+ "environment_offline",+ ))+ });+ sequence.push_response(async move {+ tokio::time::resume();+ Ok(accepted_bundle)+ });let identity = NoiseChannelIdentity::generate()?;- let _connection = ExecServerClient::open_initial_noise_rendezvous_connection(- &provider,- &identity,- codex_http_client::HttpClientFactory::new(- codex_http_client::OutboundProxyPolicy::ReqwestDefault,- ),- )- .await?;+ let _connection = sequence.connect(&identity).await?;assert_eq!(sequence.returned_urls(),vec![unauthorized_url, accepted_url]);- assert_eq!(- *sequence.requested_keys.lock().unwrap(),- vec![identity.public_key(); 4]- );+ sequence.assert_requested_identity(&identity, INITIAL_REGISTRY_MAX_RETRIES as usize + 3);unauthorized_server.await??;accepted_server.await??;Ok(())