Lightweight coding agent that runs in your terminal
Rust
Retry timed-out cloud config loads in the background (#40637)
Abhinave44bd9b64d8a
diff --git a/codex-rs/cloud-config/src/service.rs b/codex-rs/cloud-config/src/service.rsindex 4ce625eeeb..faafe895a1 100644--- a/codex-rs/cloud-config/src/service.rs+++ b/codex-rs/cloud-config/src/service.rs@@ -34,6 +34,7 @@ use tokio::time::timeout;pub(crate) const CLOUD_CONFIG_BUNDLE_TIMEOUT: Duration = Duration::from_secs(15);const CLOUD_CONFIG_BUNDLE_MAX_ATTEMPTS: usize = 5;const CLOUD_CONFIG_BUNDLE_CACHE_REFRESH_INTERVAL: Duration = Duration::from_secs(15 * 60);+const CLOUD_CONFIG_BUNDLE_TIMEOUT_RETRY_INTERVAL: Duration = Duration::from_secs(5);const CLOUD_CONFIG_BUNDLE_LOAD_FAILED_MESSAGE: &str ="Failed to load cloud config bundle (workspace-managed policies).";const CLOUD_CONFIG_BUNDLE_AUTH_RECOVERY_FAILED_MESSAGE: &str = concat!(@@ -459,7 +460,18 @@ wherepub(crate) async fn refresh_cache_in_background(&self) {loop {- sleep(CLOUD_CONFIG_BUNDLE_CACHE_REFRESH_INTERVAL).await;+ let mut refresh_interval = CLOUD_CONFIG_BUNDLE_CACHE_REFRESH_INTERVAL;+ if let Some(latest_bundle) = self.latest_bundle.get()+ && matches!(+ &*latest_bundle.lock().await,+ Err(error) if error.code() == CloudConfigBundleLoadErrorCode::Timeout+ )+ {+ // Recover startup timeouts through this worker without making+ // readers fetch concurrently or extending the startup deadline.+ refresh_interval = CLOUD_CONFIG_BUNDLE_TIMEOUT_RETRY_INTERVAL;+ }+ sleep(refresh_interval).await;match timeout(self.timeout, self.refresh_cache_once()).await {Ok(true) => {}Ok(false) => break,diff --git a/codex-rs/cloud-config/src/service_tests.rs b/codex-rs/cloud-config/src/service_tests.rsindex ff95a3ef04..d1d0c347d2 100644--- a/codex-rs/cloud-config/src/service_tests.rs+++ b/codex-rs/cloud-config/src/service_tests.rs@@ -272,6 +272,8 @@ impl BundleClient for NotifyingPendingBundleClient {struct SequenceBundleClient {responses: tokio::sync::Mutex<VecDeque<Result<CloudConfigBundle, BundleRequestError>>>,request_count: AtomicUsize,+ request_started: tokio::sync::Notify,+ timeout_attempts: usize,}impl SequenceBundleClient {@@ -279,13 +281,19 @@ impl SequenceBundleClient {Self {responses: tokio::sync::Mutex::new(VecDeque::from(responses)),request_count: AtomicUsize::new(0),+ request_started: tokio::sync::Notify::new(),+ timeout_attempts: 0,}}}impl BundleClient for SequenceBundleClient {async fn get_bundle(&self, _auth: &CodexAuth) -> Result<CloudConfigBundle, BundleRequestError> {- self.request_count.fetch_add(1, Ordering::SeqCst);+ let attempt = self.request_count.fetch_add(1, Ordering::SeqCst);+ self.request_started.notify_one();+ if attempt < self.timeout_attempts {+ pending::<()>().await;+ }let mut responses = self.responses.lock().await;responses.pop_front()@@ -1067,6 +1075,147 @@ async fn refresh_from_remote_updates_cached_bundle() {assert_eq!(signed_payload.bundle, replacement_bundle);}+#[tokio::test(start_paused = true)]+async fn production_loader_recovers_startup_timeouts_in_background() {+ let codex_home = tempdir().expect("tempdir");+ let fetcher = Arc::new(SequenceBundleClient {+ timeout_attempts: 2,+ ..SequenceBundleClient::new(vec![Ok(test_bundle())])+ });+ let service = CloudConfigBundleService::new(+ auth_manager_with_plan("business").await,+ Arc::clone(&fetcher),+ codex_home.path().to_path_buf(),+ CLOUD_CONFIG_BUNDLE_TIMEOUT,+ );+ let (loader, _) = crate::bundle_loader::cloud_config_bundle_loader_for_service(service);+ let (initial, ()) = tokio::join!(loader.get(), async {+ fetcher.request_started.notified().await;+ tokio::time::advance(CLOUD_CONFIG_BUNDLE_TIMEOUT + Duration::from_millis(1)).await;+ });+ assert_eq!(+ initial+ .as_ref()+ .expect_err("startup should time out")+ .code(),+ CloudConfigBundleLoadErrorCode::Timeout+ );+ assert_eq!(fetcher.request_count.load(Ordering::SeqCst), 1);++ for attempt in 2..=3 {+ tokio::time::timeout(+ CLOUD_CONFIG_BUNDLE_TIMEOUT_RETRY_INTERVAL + Duration::from_millis(1),+ fetcher.request_started.notified(),+ )+ .await+ .expect("the worker should retry before the normal refresh interval");+ assert_eq!(fetcher.request_count.load(Ordering::SeqCst), attempt);+ if attempt == 2 {+ // Readers keep returning the snapshot while the worker's retry is pending.+ let (first, second) = tokio::join!(loader.get(), loader.get());+ assert_eq!((first, second), (initial.clone(), initial.clone()));+ assert_eq!(fetcher.request_count.load(Ordering::SeqCst), 2);+ tokio::time::advance(CLOUD_CONFIG_BUNDLE_TIMEOUT + Duration::from_millis(1)).await;+ }+ }++ let recovery_deadline = std::time::Instant::now() + Duration::from_secs(5);+ while loader.get().await != Ok(Some(test_bundle())) {+ assert!(+ std::time::Instant::now() < recovery_deadline,+ "the worker should publish the recovered bundle"+ );+ tokio::task::yield_now().await;+ }+ let config = ConfigBuilder::default()+ .codex_home(codex_home.path().to_path_buf())+ .cloud_config_bundle(loader.clone())+ .build()+ .await+ .expect("later configuration loads should apply the recovered bundle");+ assert_eq!(config.model.as_deref(), Some("gpt-5"));+ assert_eq!(+ config.permissions.approval_policy.value(),+ codex_protocol::protocol::AskForApproval::Never+ );++ tokio::time::timeout(+ CLOUD_CONFIG_BUNDLE_TIMEOUT_RETRY_INTERVAL + Duration::from_millis(1),+ fetcher.request_started.notified(),+ )+ .await+ .expect_err("successful recovery should restore the normal refresh interval");+ tokio::time::timeout(+ CLOUD_CONFIG_BUNDLE_CACHE_REFRESH_INTERVAL,+ fetcher.request_started.notified(),+ )+ .await+ .expect("normal background refresh should continue after recovery");+ assert_eq!(fetcher.request_count.load(Ordering::SeqCst), 4);+}++#[tokio::test(start_paused = true)]+async fn production_loader_restores_normal_refresh_interval_after_non_timeout_error() {+ let codex_home = tempdir().expect("tempdir");+ let fetcher = Arc::new(SequenceBundleClient {+ timeout_attempts: 1,+ ..SequenceBundleClient::new(vec![Ok(invalid_config_bundle()), Ok(test_bundle())])+ });+ let service = CloudConfigBundleService::new(+ auth_manager_with_plan("business").await,+ Arc::clone(&fetcher),+ codex_home.path().to_path_buf(),+ CLOUD_CONFIG_BUNDLE_TIMEOUT,+ );+ let (loader, _) = crate::bundle_loader::cloud_config_bundle_loader_for_service(service);+ let (initial, ()) = tokio::join!(loader.get(), async {+ fetcher.request_started.notified().await;+ tokio::time::advance(CLOUD_CONFIG_BUNDLE_TIMEOUT + Duration::from_millis(1)).await;+ });+ assert_eq!(+ initial.expect_err("startup should time out").code(),+ CloudConfigBundleLoadErrorCode::Timeout+ );+ assert_eq!(fetcher.request_count.load(Ordering::SeqCst), 1);++ tokio::time::timeout(+ CLOUD_CONFIG_BUNDLE_TIMEOUT_RETRY_INTERVAL + Duration::from_millis(1),+ fetcher.request_started.notified(),+ )+ .await+ .expect("the worker should retry the startup timeout promptly");+ let refresh_deadline = std::time::Instant::now() + Duration::from_secs(5);+ loop {+ let error = loader+ .get()+ .await+ .expect_err("the invalid bundle should fail closed");+ if error.code() == CloudConfigBundleLoadErrorCode::InvalidBundle {+ break;+ }+ assert!(+ std::time::Instant::now() < refresh_deadline,+ "the retry should replace the cached timeout with the validation error"+ );+ tokio::task::yield_now().await;+ }+ assert_eq!(fetcher.request_count.load(Ordering::SeqCst), 2);++ tokio::time::timeout(+ CLOUD_CONFIG_BUNDLE_CACHE_REFRESH_INTERVAL - Duration::from_secs(1),+ fetcher.request_started.notified(),+ )+ .await+ .expect_err("the non-timeout error should stop fast retries");+ tokio::time::timeout(+ Duration::from_secs(1) + Duration::from_millis(1),+ fetcher.request_started.notified(),+ )+ .await+ .expect("refresh should resume at the normal interval");+ assert_eq!(fetcher.request_count.load(Ordering::SeqCst), 3);+}+#[tokio::test(start_paused = true)]async fn production_loader_refreshes_later_configs_and_preserves_failed_refreshes() {let codex_home = tempdir().expect("tempdir");