Skip to content

armgit / codex

Public Imported from GitHub
0

Lightweight coding agent that runs in your terminal

Rust

Allow provisioned executors more time to come online (#48575)

richardopenai985cf47a4eb6
diff --git a/codex-rs/exec-server/src/client.rs b/codex-rs/exec-server/src/client.rs
index dcb05293fb..538bb214fc 100644
--- a/codex-rs/exec-server/src/client.rs
+++ b/codex-rs/exec-server/src/client.rs
@@ -150,7 +150,6 @@ mod recovery;
#[path = "client_refresh.rs"]
mod refresh;
pub(crate) use connection_failure::can_retry_connection_attempt;
-#[cfg(test)]
pub(crate) use recovery::is_environment_offline_error;
pub(crate) use recovery::is_retryable_recovery_error;
diff --git a/codex-rs/exec-server/src/client_refresh_tests.rs b/codex-rs/exec-server/src/client_refresh_tests.rs
index 22bcb30679..26d4f2ad67 100644
--- a/codex-rs/exec-server/src/client_refresh_tests.rs
+++ b/codex-rs/exec-server/src/client_refresh_tests.rs
@@ -1,5 +1,8 @@
use std::sync::Arc;
use std::sync::Mutex;
+use std::sync::atomic::AtomicBool;
+use std::sync::atomic::AtomicUsize;
+use std::sync::atomic::Ordering;
use std::time::Duration;
use anyhow::Result;
@@ -49,6 +52,8 @@ struct Registry {
target: Mutex<Target>,
next_lookup: Mutex<Option<oneshot::Receiver<()>>>,
lookup_started: Notify,
+ offline: AtomicBool,
+ calls: AtomicUsize,
}
impl Registry {
@@ -65,6 +70,14 @@ impl NoiseRendezvousConnectProvider for Registry {
_: NoiseChannelPublicKey,
) -> BoxFuture<'_, Result<NoiseRendezvousConnectBundle, ExecServerError>> {
Box::pin(async move {
+ self.calls.fetch_add(1, Ordering::Relaxed);
+ if self.offline.load(Ordering::Relaxed) {
+ return Err(ExecServerError::EnvironmentRegistryHttp {
+ status: http::StatusCode::CONFLICT,
+ code: Some("environment_offline".to_owned()),
+ message: "executor is still starting".to_owned(),
+ });
+ }
let target = self.target.lock().unwrap().clone();
let block = self.next_lookup.lock().unwrap().take();
if let Some(block) = block {
@@ -203,6 +216,8 @@ impl Executor {
target: Mutex::new(self.target.clone()),
next_lookup: Mutex::new(None),
lookup_started: Notify::new(),
+ offline: AtomicBool::new(false),
+ calls: AtomicUsize::new(0),
});
let client = LazyRemoteExecServerClient::new(
ExecServerTransportParams::NoiseRendezvous {
@@ -635,6 +650,63 @@ async fn superseded_refresh_lookup_does_not_retire_a_newer_session() -> Result<(
Ok(())
}
+#[tokio::test]
+async fn provisioned_environment_waits_for_offline_executor_on_the_same_handle() -> Result<()> {
+ let _clock = freeze_clock();
+ let executor = Executor::start(Validator::default()).await?;
+ let (_, registry) = executor.client()?;
+ registry.offline.store(true, Ordering::Relaxed);
+ let manager = crate::EnvironmentManager::from_snapshot(
+ crate::environment_provider::EnvironmentProviderSnapshot {
+ environments: Vec::new(),
+ default: crate::environment_provider::EnvironmentDefault::Disabled,
+ include_local: false,
+ },
+ /*local_runtime_paths*/ None,
+ HttpClientFactory::new(OutboundProxyPolicy::ReqwestDefault),
+ )?;
+ let environment = manager
+ .report_environment_provisioning_status(
+ "environment".to_owned(),
+ Ok(crate::EnvironmentReadyInfo::default()),
+ registry.clone(),
+ )?
+ .expect("provisioned environment");
+ let ready = environment.wait_until_ready();
+ let info = environment.info();
+ tokio::pin!(ready, info);
+ assert!(futures::poll!(ready.as_mut()).is_pending());
+ assert!(futures::poll!(info.as_mut()).is_pending());
+ assert_eq!(registry.calls.load(Ordering::Relaxed), 1);
+
+ // A restore can outlast both the ordinary registry retry count and deadline.
+ for _ in 0..35 {
+ tokio::time::advance(Duration::from_secs(1)).await;
+ assert!(futures::poll!(ready.as_mut()).is_pending());
+ assert!(futures::poll!(info.as_mut()).is_pending());
+ }
+ assert!(registry.calls.load(Ordering::Relaxed) > 5);
+ let reported = manager
+ .report_environment_provisioning_status(
+ "environment".to_owned(),
+ Ok(crate::EnvironmentReadyInfo::default()),
+ registry.clone(),
+ )?
+ .expect("same provisioned environment");
+ assert!(Arc::ptr_eq(&environment, &reported));
+ assert!(futures::poll!(ready.as_mut()).is_pending());
+ assert!(futures::poll!(info.as_mut()).is_pending());
+
+ registry.offline.store(false, Ordering::Relaxed);
+ tokio::time::advance(Duration::from_secs(8)).await;
+ let ((), _) = tokio::try_join!(ready, info)?;
+ assert_eq!(
+ environment.cached_executor_registration_id(),
+ Some(executor.target.registration.clone())
+ );
+ Ok(())
+}
+
#[tokio::test]
async fn environment_refresh_preserves_environment_and_filesystem_handles() -> Result<()> {
let _clock = freeze_clock();
diff --git a/codex-rs/exec-server/src/client_transport.rs b/codex-rs/exec-server/src/client_transport.rs
index a6867f2a27..a8a5d7967e 100644
--- a/codex-rs/exec-server/src/client_transport.rs
+++ b/codex-rs/exec-server/src/client_transport.rs
@@ -31,6 +31,7 @@ use crate::ExecServerClient;
use crate::ExecServerError;
use crate::client::NoiseInitializeContext;
use crate::client::accepted::AcceptedConnectionSource;
+use crate::client::is_environment_offline_error;
use crate::client::is_retryable_registry_error;
use crate::client::registry_recovery_retry_delay;
use crate::client_api::DEFAULT_REMOTE_EXEC_SERVER_CONNECT_TIMEOUT;
@@ -56,6 +57,7 @@ const ENVIRONMENT_CLIENT_NAME: &str = "codex-environment";
const INITIAL_REGISTRY_MAX_RETRIES: u32 = 4;
const INITIAL_REGISTRY_REQUEST_TIMEOUT: Duration = Duration::from_secs(6);
const INITIAL_REGISTRY_OPERATION_TIMEOUT: Duration = Duration::from_secs(14);
+const PROVISIONED_ENVIRONMENT_CONNECT_TIMEOUT: Duration = Duration::from_secs(5 * 60);
pub(crate) async fn connect_websocket_request(
request: http::Request<()>,
@@ -337,7 +339,7 @@ impl ExecServerClient {
transport_params => (transport_params, None),
};
- if let Some(mut readiness) = deferred_readiness {
+ let provisioning_deadline = if let Some(mut readiness) = deferred_readiness {
let provisioning_result = readiness
.wait_for(Option::is_some)
.await
@@ -355,7 +357,10 @@ impl ExecServerClient {
)
})?;
provisioning_result.map_err(ExecServerError::ProvisioningFailed)?;
- }
+ Some(Instant::now() + PROVISIONED_ENVIRONMENT_CONNECT_TIMEOUT)
+ } else {
+ None
+ };
let websocket = match transport_params {
ExecServerTransportParams::Deferred(_) => {
@@ -379,6 +384,7 @@ impl ExecServerClient {
&provider,
&identity,
http_client_factory.clone(),
+ provisioning_deadline,
)
.await?;
let reconnect_strategy = ExecServerReconnectStrategy::NoiseRendezvous {
@@ -431,6 +437,7 @@ impl ExecServerClient {
provider: &Arc<dyn NoiseRendezvousConnectProvider>,
identity: &NoiseChannelIdentity,
http_client_factory: HttpClientFactory,
+ provisioning_deadline: Option<Instant>,
) -> Result<(ReadyNoiseRendezvousConnection, crate::NoiseChannelPublicKey), ExecServerError>
{
let open_connection = |bundle: NoiseRendezvousConnectBundle| {
@@ -464,14 +471,22 @@ impl ExecServerClient {
loop {
let bundle = match result {
Ok(bundle) => bundle,
- Err(error)
- if is_retryable_registry_error(&error)
- && retries < INITIAL_REGISTRY_MAX_RETRIES =>
- {
+ Err(error) => {
+ // A provisioned executor may be resuming after an earlier ready
+ // report. Only offline responses get the longer, fixed deadline.
+ let retry_deadline = match provisioning_deadline {
+ Some(deadline) if is_environment_offline_error(&error) => deadline,
+ _ if is_retryable_registry_error(&error)
+ && retries < INITIAL_REGISTRY_MAX_RETRIES =>
+ {
+ deadline
+ }
+ _ => return Err(error),
+ };
// Session resumption owns its separate recovery deadline.
let delay = registry_recovery_retry_delay(&retry_key, retries);
retries += 1;
- result = match timeout_at(deadline, async {
+ result = match timeout_at(retry_deadline, async {
sleep(delay).await;
connect_bundle().await
})
@@ -482,7 +497,6 @@ impl ExecServerClient {
};
continue;
}
- Err(error) => return Err(error),
};
let executor_public_key = bundle.executor_public_key.clone();
match open_connection(bundle).await {
diff --git a/codex-rs/exec-server/src/client_transport_tests.rs b/codex-rs/exec-server/src/client_transport_tests.rs
index 6f66c13d32..fe1966f8fc 100644
--- a/codex-rs/exec-server/src/client_transport_tests.rs
+++ b/codex-rs/exec-server/src/client_transport_tests.rs
@@ -24,6 +24,7 @@ use super::ExecServerReconnectStrategy;
use super::INITIAL_REGISTRY_MAX_RETRIES;
use super::INITIAL_REGISTRY_OPERATION_TIMEOUT;
use super::INITIAL_REGISTRY_REQUEST_TIMEOUT;
+use super::PROVISIONED_ENVIRONMENT_CONNECT_TIMEOUT;
use crate::ExecServerError;
use crate::NoiseChannelIdentity;
use crate::NoiseChannelPublicKey;
@@ -33,7 +34,9 @@ use crate::NoiseRendezvousConnectProvider;
use crate::client::NoiseInitializeContext;
use crate::client_api::DEFAULT_REMOTE_EXEC_SERVER_CONNECT_TIMEOUT;
use crate::client_api::DEFAULT_REMOTE_EXEC_SERVER_INITIALIZE_TIMEOUT;
+use crate::client_api::Deferred;
use crate::client_api::ExecServerClientConnectOptions;
+use crate::client_api::ExecServerTransportParams;
use crate::connection::JsonRpcConnection;
use crate::noise_channel::PendingResponderHandshake;
use crate::noise_channel::noise_channel_prologue;
@@ -101,10 +104,31 @@ impl SequenceNoiseConnectProvider {
codex_http_client::HttpClientFactory::new(
codex_http_client::OutboundProxyPolicy::ReqwestDefault,
),
+ /*provisioning_deadline*/ None,
)
.await
.map(|(ready, _)| (ready.connection, ready.options))
}
+
+ async fn connect_provisioned(
+ self: &Arc<Self>,
+ identity: &NoiseChannelIdentity,
+ ) -> Result<ExecServerClient, ExecServerError> {
+ let (_readiness_tx, readiness) = tokio::sync::watch::channel(Some(Ok(())));
+ ExecServerClient::connect_for_transport(
+ ExecServerTransportParams::Deferred(Box::new(Deferred {
+ readiness,
+ transport: ExecServerTransportParams::NoiseRendezvous {
+ provider: self.clone(),
+ identity: identity.clone(),
+ },
+ })),
+ codex_http_client::HttpClientFactory::new(
+ codex_http_client::OutboundProxyPolicy::ReqwestDefault,
+ ),
+ )
+ .await
+ }
}
impl NoiseRendezvousConnectProvider for SequenceNoiseConnectProvider {
@@ -455,6 +479,124 @@ async fn initial_noise_connection_does_not_retry_permanent_registry_errors() ->
Ok(())
}
+#[tokio::test(start_paused = true)]
+async fn provisioned_noise_connection_bounds_offline_retries() -> Result<()> {
+ let sequence = Arc::new(SequenceNoiseConnectProvider::default());
+ // Enough responses to reach the deadline even at the shortest retry interval.
+ for _ in 0..1024 {
+ sequence.push_error(registry_error(
+ http::StatusCode::CONFLICT,
+ "environment_offline",
+ ));
+ }
+ let identity = NoiseChannelIdentity::generate()?;
+ let started = tokio::time::Instant::now();
+
+ let error = sequence
+ .connect_provisioned(&identity)
+ .await
+ .err()
+ .expect("a provisioned executor that stays offline must time out");
+
+ assert!(crate::client::is_environment_offline_error(&error));
+ assert_eq!(started.elapsed(), PROVISIONED_ENVIRONMENT_CONNECT_TIMEOUT);
+ let requests = sequence.requested_keys().len();
+ assert!(requests > INITIAL_REGISTRY_MAX_RETRIES as usize + 1);
+ sequence.assert_requested_identity(&identity, requests);
+ Ok(())
+}
+
+#[tokio::test(start_paused = true)]
+async fn provisioned_noise_connection_keeps_other_registry_retry_limits() -> Result<()> {
+ let sequence = Arc::new(SequenceNoiseConnectProvider::default());
+ for _ in 0..=INITIAL_REGISTRY_MAX_RETRIES {
+ sequence.push_error(registry_error(
+ http::StatusCode::SERVICE_UNAVAILABLE,
+ "unavailable",
+ ));
+ }
+ let identity = NoiseChannelIdentity::generate()?;
+ let started = tokio::time::Instant::now();
+
+ let error = sequence
+ .connect_provisioned(&identity)
+ .await
+ .err()
+ .expect("only an offline executor receives the provisioning grace");
+
+ assert!(matches!(
+ error,
+ ExecServerError::EnvironmentRegistryHttp {
+ status: http::StatusCode::SERVICE_UNAVAILABLE,
+ ..
+ }
+ ));
+ assert!(started.elapsed() <= INITIAL_REGISTRY_OPERATION_TIMEOUT);
+ assert!(sequence.requested_keys().len() <= INITIAL_REGISTRY_MAX_RETRIES as usize + 1);
+ Ok(())
+}
+
+#[tokio::test(start_paused = true)]
+async fn provisioned_noise_connection_bounds_a_stalled_retry_request() -> Result<()> {
+ let sequence = Arc::new(SequenceNoiseConnectProvider::default());
+ sequence.push_error(registry_error(
+ http::StatusCode::CONFLICT,
+ "environment_offline",
+ ));
+ for _ in 0..INITIAL_REGISTRY_MAX_RETRIES {
+ sequence.push_pending();
+ }
+ let identity = NoiseChannelIdentity::generate()?;
+ let started = tokio::time::Instant::now();
+ let error = sequence
+ .connect_provisioned(&identity)
+ .await
+ .err()
+ .expect("a stalled registry request must keep the ordinary timeout");
+
+ assert!(matches!(
+ error,
+ ExecServerError::EnvironmentRegistryRequest(error) if error.is_timeout()
+ ));
+ assert_eq!(started.elapsed(), INITIAL_REGISTRY_OPERATION_TIMEOUT);
+ let requests = sequence.requested_keys().len();
+ assert!((2..=3).contains(&requests));
+ sequence.assert_requested_identity(&identity, requests);
+ Ok(())
+}
+
+#[tokio::test(start_paused = true)]
+async fn provisioned_noise_connection_stops_on_a_late_permanent_error() -> Result<()> {
+ let sequence = Arc::new(SequenceNoiseConnectProvider::default());
+ for _ in 0..10 {
+ sequence.push_error(registry_error(
+ http::StatusCode::CONFLICT,
+ "environment_offline",
+ ));
+ }
+ sequence.push_error(registry_error(http::StatusCode::FORBIDDEN, "forbidden"));
+ let identity = NoiseChannelIdentity::generate()?;
+ let started = tokio::time::Instant::now();
+
+ let error = sequence
+ .connect_provisioned(&identity)
+ .await
+ .err()
+ .expect("a permanent error must stop provisioning retries");
+
+ assert!(matches!(
+ error,
+ ExecServerError::EnvironmentRegistryHttp {
+ status: http::StatusCode::FORBIDDEN,
+ code: Some(code),
+ ..
+ } if code == "forbidden"
+ ));
+ sequence.assert_requested_identity(&identity, /*requests*/ 11);
+ assert!(started.elapsed() > INITIAL_REGISTRY_OPERATION_TIMEOUT);
+ Ok(())
+}
+
#[tokio::test]
async fn noise_session_resume_rejects_a_changed_noise_key_before_connecting() -> Result<()> {
let sequence = Arc::new(SequenceNoiseConnectProvider::default());