Skip to content

armgit / codex

Public Imported from GitHub
0

Lightweight coding agent that runs in your terminal

Rust

Back off failed cloud skill and plugin catalog refreshes (#55968)

Anthony Tafoya3fd08b540570
diff --git a/codex-rs/core/tests/suite/skills_extension/cloud_lifecycle_tests.rs b/codex-rs/core/tests/suite/skills_extension/cloud_lifecycle_tests.rs
index 32642b6628..96e7375281 100644
--- a/codex-rs/core/tests/suite/skills_extension/cloud_lifecycle_tests.rs
+++ b/codex-rs/core/tests/suite/skills_extension/cloud_lifecycle_tests.rs
@@ -132,7 +132,7 @@ async fn cloud_skills_reuse_cache_and_invalidate_on_connection_or_auth_change()
});
let test = builder.build_with_auto_env(&server).await?;
let mut last_available_skill = calendar;
- // Retry turn-start failures before each model step.
+ // Back off failures across steps and turns; reconnects allow immediate recovery.
for (turn, next_reply, expected_skill, new_account, expected_lists, expected_reads) in [
("discovered", Ok(Some(calendar)), Some(calendar), None, 1, 1),
(
@@ -156,7 +156,7 @@ async fn cloud_skills_reuse_cache_and_invalidate_on_connection_or_auth_change()
Err(()),
Some(updated_calendar),
None,
- 3,
+ 1,
0,
),
("removed", Ok(None), None, None, 1, 0),
@@ -174,9 +174,10 @@ async fn cloud_skills_reuse_cache_and_invalidate_on_connection_or_auth_change()
Err(()),
None,
Some("account-b"),
- 3,
+ 1,
0,
),
+ ("new-account-cooldown", Ok(Some(gmail)), None, None, 0, 0),
(
"new-account-discovered",
Ok(Some(gmail)),
@@ -190,7 +191,7 @@ async fn cloud_skills_reuse_cache_and_invalidate_on_connection_or_auth_change()
Err(()),
Some(gmail),
None,
- 3,
+ 1,
0,
),
(
@@ -207,7 +208,13 @@ async fn cloud_skills_reuse_cache_and_invalidate_on_connection_or_auth_change()
*provider.reply.lock().expect("cloud reply lock") = next_reply;
let previous_connection_scope = if matches!(
turn,
- "refreshed" | "same-account-failure" | "restored" | "connection-replaced-failure"
+ "refreshed"
+ | "same-account-failure"
+ | "removed"
+ | "restored"
+ | "new-account-discovered"
+ | "connection-replaced-failure"
+ | "replacement-connection-discovered"
) {
let client = resources
.0
diff --git a/codex-rs/ext/extension-api/Cargo.toml b/codex-rs/ext/extension-api/Cargo.toml
index 2c2bd6d384..3b80882c4a 100644
--- a/codex-rs/ext/extension-api/Cargo.toml
+++ b/codex-rs/ext/extension-api/Cargo.toml
@@ -29,6 +29,7 @@ codex-tools = { workspace = true }
codex-utils-absolute-path = { workspace = true }
codex-utils-path-uri = { workspace = true }
serde_json = { workspace = true }
+tokio = { workspace = true, features = ["time"] }
[dev-dependencies]
pretty_assertions = { workspace = true }
diff --git a/codex-rs/ext/extension-api/src/catalog_refresh.rs b/codex-rs/ext/extension-api/src/catalog_refresh.rs
new file mode 100644
index 0000000000..b5bb07ccd8
--- /dev/null
+++ b/codex-rs/ext/extension-api/src/catalog_refresh.rs
@@ -0,0 +1,55 @@
+//! Thread-owned cloud catalog retry cooldowns scoped to auth and resource generations.
+
+use codex_mcp::McpResourceClientAuthKey;
+use codex_mcp::McpResourceServerCacheKey;
+use std::time::Duration;
+use tokio::time::Instant;
+
+/// Store per catalog and thread; clear on success or configuration reset.
+#[derive(Clone)]
+pub struct CatalogRefreshBackoff {
+ auth: Option<McpResourceClientAuthKey>,
+ resources: Option<McpResourceServerCacheKey>,
+ delay: Duration,
+ retry_at: Instant,
+}
+
+impl std::fmt::Debug for CatalogRefreshBackoff {
+ fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ formatter
+ .debug_struct("CatalogRefreshBackoff")
+ .field("delay", &self.delay)
+ .field("retry_at", &self.retry_at)
+ .finish_non_exhaustive()
+ }
+}
+
+impl CatalogRefreshBackoff {
+ pub fn is_pending(
+ &self,
+ auth: Option<&McpResourceClientAuthKey>,
+ resources: Option<&McpResourceServerCacheKey>,
+ ) -> bool {
+ self.auth.as_ref() == auth
+ && self.resources.as_ref() == resources
+ && Instant::now() < self.retry_at
+ }
+
+ pub fn after_failure(
+ auth: Option<McpResourceClientAuthKey>,
+ resources: Option<McpResourceServerCacheKey>,
+ previous: Option<&Self>,
+ ) -> Self {
+ let delay = previous
+ .filter(|previous| previous.auth == auth && previous.resources == resources)
+ .map_or(Duration::from_secs(30), |previous| {
+ (previous.delay * 2).min(Duration::from_secs(300))
+ });
+ Self {
+ auth,
+ resources,
+ delay,
+ retry_at: Instant::now() + delay,
+ }
+ }
+}
diff --git a/codex-rs/ext/extension-api/src/lib.rs b/codex-rs/ext/extension-api/src/lib.rs
index f66a2ee0db..eabf26998b 100644
--- a/codex-rs/ext/extension-api/src/lib.rs
+++ b/codex-rs/ext/extension-api/src/lib.rs
@@ -1,4 +1,5 @@
mod capabilities;
+mod catalog_refresh;
mod contributors;
mod message_lifecycle;
mod registry;
@@ -22,6 +23,7 @@ pub use capabilities::NoopExtensionEventSink;
pub use capabilities::NoopResponseItemInjector;
pub use capabilities::ResponseItemInjectionFuture;
pub use capabilities::ResponseItemInjector;
+pub use catalog_refresh::CatalogRefreshBackoff;
pub use codex_context_fragments::ContextualUserFragment;
pub use codex_mcp::McpProtocolMode;
pub use codex_mcp::ToolInfo as McpToolInfo;
diff --git a/codex-rs/ext/mcp/Cargo.toml b/codex-rs/ext/mcp/Cargo.toml
index 5185f6cab1..a30517a3f8 100644
--- a/codex-rs/ext/mcp/Cargo.toml
+++ b/codex-rs/ext/mcp/Cargo.toml
@@ -40,6 +40,6 @@ codex-login = { workspace = true }
futures = { workspace = true }
pretty_assertions = { workspace = true }
tempfile = { workspace = true }
-tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }
+tokio = { workspace = true, features = ["macros", "rt-multi-thread", "test-util"] }
tokio-util = { workspace = true, features = ["rt"] }
wiremock = { workspace = true }
diff --git a/codex-rs/ext/mcp/src/cloud_plugin.rs b/codex-rs/ext/mcp/src/cloud_plugin.rs
index 035ef367c2..4ef00874f9 100644
--- a/codex-rs/ext/mcp/src/cloud_plugin.rs
+++ b/codex-rs/ext/mcp/src/cloud_plugin.rs
@@ -6,6 +6,7 @@ use crate::plugin_contributor::PluginContributor;
use codex_core::config::Config;
use codex_core_plugins::PluginCatalog;
use codex_core_plugins::PluginIdentity;
+use codex_extension_api::CatalogRefreshBackoff;
use codex_extension_api::CatalogRefreshInput;
use codex_extension_api::ConfigContributor;
use codex_extension_api::ExtensionData;
@@ -52,49 +53,60 @@ impl PluginContributor {
mcp_resources: input.session_store.get::<McpResourceClient>(),
snapshots: Vec::new(),
};
- let (auth_cache_key, catalog_cleared) = {
- let mut state = state.contributor_state();
- if !state.plugins_enabled {
- return false;
- }
- let auth = query
- .mcp_resources
- .as_ref()
- .map(|client| client.auth_cache_key_for_server(CODEX_APPS_MCP_SERVER_NAME));
- if ((waited && *refresh_succeeded)
- || input
- .turn_store
- .get::<LatestCloudPluginAuth>()
- .is_some_and(|previous| previous.0 == auth))
- && state
+ let (auth_cache_key, resource_cache_key, catalog_cleared) =
+ {
+ let mut state = state.contributor_state();
+ if !state.plugins_enabled {
+ return false;
+ }
+ let auth = query
+ .mcp_resources
+ .as_ref()
+ .map(|client| client.auth_cache_key_for_server(CODEX_APPS_MCP_SERVER_NAME));
+ let resource_cache_key = query
+ .mcp_resources
+ .as_ref()
+ .and_then(|client| client.server_cache_key(CODEX_APPS_MCP_SERVER_NAME));
+ if ((waited && *refresh_succeeded)
+ || input
+ .turn_store
+ .get::<LatestCloudPluginAuth>()
+ .is_some_and(|previous| previous.0 == auth))
+ && state
+ .cloud_generation
+ .as_ref()
+ .is_some_and(|generation| generation.auth_cache_key == auth)
+ && state.cloud_catalog().is_some()
+ {
+ input.turn_store.insert(LatestCloudPluginAuth(auth));
+ return false;
+ }
+ if state.cloud_retry.as_ref().is_some_and(|retry| {
+ retry.is_pending(auth.as_ref(), resource_cache_key.as_ref())
+ }) {
+ return false;
+ }
+ // Waiting callers must not reuse a failed or cancelled refresh.
+ *refresh_succeeded = false;
+ input.turn_store.remove::<LatestCloudPluginAuth>();
+ let previous_catalog_auth = state
.cloud_generation
.as_ref()
- .is_some_and(|generation| generation.auth_cache_key == auth)
- && state.cloud_catalog().is_some()
- {
- input.turn_store.insert(LatestCloudPluginAuth(auth));
- return false;
- }
- // A failed or cancelled refresh must leave waiting callers able to retry.
- *refresh_succeeded = false;
- input.turn_store.remove::<LatestCloudPluginAuth>();
- let previous_catalog_auth = state
- .cloud_generation
- .as_ref()
- .filter(|generation| generation.catalog.is_some())
- .map(|generation| generation.auth_cache_key.clone());
- let auth_cache_key =
- state.prepare_cloud_generation(query.mcp_resources.as_deref().cloned());
- let catalog_cleared =
- previous_catalog_auth.is_some_and(|previous| previous != auth_cache_key);
- (auth_cache_key, catalog_cleared)
- };
+ .filter(|generation| generation.catalog.is_some())
+ .map(|generation| generation.auth_cache_key.clone());
+ let auth_cache_key =
+ state.prepare_cloud_generation(query.mcp_resources.as_deref().cloned());
+ let catalog_cleared =
+ previous_catalog_auth.is_some_and(|previous| previous != auth_cache_key);
+ (auth_cache_key, resource_cache_key, catalog_cleared)
+ };
let result = provider.list(query).await;
let mut state = state.contributor_state();
let catalog_published = match result {
Ok(catalog) => {
let published = state.publish_cloud_catalog(auth_cache_key.as_ref(), catalog);
if published {
+ state.cloud_retry = None;
input
.turn_store
.insert(LatestCloudPluginAuth(auth_cache_key));
@@ -105,6 +117,15 @@ impl PluginContributor {
published
}
Err(_) => {
+ if state.cloud_generation.as_ref().is_some_and(|generation| {
+ generation.auth_cache_key == auth_cache_key && generation.is_auth_current()
+ }) {
+ state.cloud_retry = Some(CatalogRefreshBackoff::after_failure(
+ auth_cache_key,
+ resource_cache_key,
+ state.cloud_retry.as_ref(),
+ ));
+ }
tracing::warn!("cloud plugin discovery failed");
false
}
@@ -140,9 +161,13 @@ impl ConfigContributor<Config> for PluginContributor {
let plugins_enabled = new_config.features.enabled(Feature::Plugins);
let state = thread_store.get_or_init(PluginsThreadState::default);
let mut state = state.contributor_state();
+ if state.plugins_enabled != plugins_enabled {
+ state.cloud_retry = None;
+ }
state.plugins_enabled = plugins_enabled;
if self.providers.cloud.is_none() || !plugins_enabled {
state.cloud_generation = None;
+ state.cloud_retry = None;
}
}
}
diff --git a/codex-rs/ext/mcp/src/cloud_plugin_tests.rs b/codex-rs/ext/mcp/src/cloud_plugin_tests.rs
index 4c592b248a..23a6809315 100644
--- a/codex-rs/ext/mcp/src/cloud_plugin_tests.rs
+++ b/codex-rs/ext/mcp/src/cloud_plugin_tests.rs
@@ -130,6 +130,7 @@ async fn disabling_plugins_clears_cloud_catalog_and_skips_discovery() -> anyhow:
true,
Err("discovery unavailable".to_string()),
),
+ ("failed-plugins-off", false, Ok(cloud_catalog())),
("re-enabled", true, Ok(cloud_catalog())),
] {
let expected_catalog = if plugins_enabled {
@@ -245,7 +246,7 @@ async fn account_change_clears_hosted_plugins_before_waiting_for_executor() -> a
Ok(())
}
-#[tokio::test]
+#[tokio::test(start_paused = true)]
async fn cloud_refresh_serializes_waiters_and_retries() {
let new_catalog = PluginCatalog::default();
for (reply, cancelled) in [
@@ -303,13 +304,19 @@ async fn cloud_refresh_serializes_waiters_and_retries() {
provider.resume_discovery.notify_one();
assert_eq!(turn_start.await, succeeded);
}
- // Reuse a successful refresh; failure/cancellation releases the gate for retry.
+ // Waiting turns share failure cooldowns; cancellation still allows immediate retry.
assert_eq!(
tokio::time::timeout(Duration::from_secs(5), prewarm)
.await
.expect("waiting discovery should complete after its predecessor stops"),
- !succeeded,
+ cancelled,
);
+ if !succeeded && !cancelled {
+ assert_eq!(state.cloud_catalog(), None);
+ assert_eq!(provider.calls.load(Ordering::SeqCst), 1);
+ tokio::time::advance(Duration::from_secs(30)).await;
+ assert!(contributor.refresh_catalog_for_step(prewarm_input).await);
+ }
assert_eq!(state.cloud_catalog(), Some(expected_catalog.clone()));
assert!(!contributor.refresh_catalog_for_step(prewarm_input).await);
assert_eq!(
@@ -325,7 +332,7 @@ async fn cloud_refresh_serializes_waiters_and_retries() {
assert!(!contributor.refresh_catalog_for_step(prewarm_input).await);
assert_eq!(state.cloud_catalog(), Some(expected_catalog.clone()));
}
- assert_eq!(provider.calls.load(Ordering::SeqCst), previous_calls + 2);
+ assert_eq!(provider.calls.load(Ordering::SeqCst), previous_calls + 1);
// Clearing an old auth scope still requires rebuilding, even on failure.
session.insert(McpResourceClient::new(Arc::new(
@@ -335,6 +342,7 @@ async fn cloud_refresh_serializes_waiters_and_retries() {
assert_eq!(state.cloud_catalog(), None);
assert!(!contributor.refresh_catalog_for_step(prewarm_input).await);
provider.set_reply(Ok(new_catalog.clone()));
+ tokio::time::advance(Duration::from_secs(30)).await;
assert!(contributor.refresh_catalog_for_step(prewarm_input).await);
}
}
@@ -660,7 +668,7 @@ async fn run_cloud_catalog_refresh_lifecycle() -> anyhow::Result<()> {
),
] {
let previous_discoveries = provider.calls.load(Ordering::SeqCst);
- let retry_discovery = reply.is_err() || turn == "in-flight-auth-change";
+ let retry_discovery = turn == "in-flight-auth-change";
if let Some(account) = new_account {
// Do not manually refresh MCP: the auth transition must pick up this new inventory.
*apps_tools
@@ -700,6 +708,15 @@ async fn run_cloud_catalog_refresh_lifecycle() -> anyhow::Result<()> {
.await?;
assert_eq!(provider.calls.load(Ordering::SeqCst), previous_discoveries);
}
+ if matches!(
+ turn,
+ "complete-with-warning-disables-plugin" | "new-account-partial"
+ ) {
+ // Reconnecting permits recovery before the thread's cooldown expires.
+ test.codex
+ .submit(codex_protocol::protocol::Op::RefreshMcpServers)
+ .await?;
+ }
provider.set_reply(reply);
let previous_calls = recorded_apps_tool_calls(&server).await.len();
let call_id = format!("{turn}-call");
diff --git a/codex-rs/ext/mcp/src/plugin_contributor_state.rs b/codex-rs/ext/mcp/src/plugin_contributor_state.rs
index 83d2e059c7..db07aefce9 100644
--- a/codex-rs/ext/mcp/src/plugin_contributor_state.rs
+++ b/codex-rs/ext/mcp/src/plugin_contributor_state.rs
@@ -5,6 +5,7 @@ use codex_core_plugins::PluginRootOwnership;
use codex_core_plugins::catalog_from_snapshots;
use codex_exec_server::ExecutorCapabilityDiscoveryV2Snapshot;
use codex_exec_server::FileSystemSandboxContext;
+use codex_extension_api::CatalogRefreshBackoff;
use codex_extension_api::SelectedPluginContribution;
use codex_mcp::CODEX_APPS_MCP_SERVER_NAME;
use codex_mcp::McpResourceClient;
@@ -41,6 +42,7 @@ impl PluginsThreadState {
pub(crate) struct PluginContributorState {
pub(crate) plugins_enabled: bool,
pub(crate) cloud_generation: Option<CloudPluginGeneration>,
+ pub(crate) cloud_retry: Option<CatalogRefreshBackoff>,
pub(crate) executor_catalog: Option<PluginCatalog>,
executor_snapshots: Vec<ExecutorCapabilityDiscoveryV2Snapshot>,
pub(crate) executor_cache: Vec<CachedSelectedRoot>,
diff --git a/codex-rs/ext/mcp/tests/plugin_mcp.rs b/codex-rs/ext/mcp/tests/plugin_mcp.rs
index 138eca6123..d9b5a99def 100644
--- a/codex-rs/ext/mcp/tests/plugin_mcp.rs
+++ b/codex-rs/ext/mcp/tests/plugin_mcp.rs
@@ -908,16 +908,10 @@ async fn run_cloud_plugin_projection_scenarios() -> anyhow::Result<()> {
initialized, installed,
"{case}: cloud-only declarations must not start MCPs"
);
+ // Failed discovery stays in cooldown across both model steps.
assert_eq!(
cloud.calls.load(Ordering::SeqCst),
- if !cloud_enabled {
- 0
- } else if cloud_available {
- 1
- } else {
- // Failed turn-start discovery retries before each model step.
- 1 + requests.len()
- },
+ usize::from(cloud_enabled),
"{case}"
);
let state = test
diff --git a/codex-rs/ext/skills/Cargo.toml b/codex-rs/ext/skills/Cargo.toml
index fa194ca760..ee67976705 100644
--- a/codex-rs/ext/skills/Cargo.toml
+++ b/codex-rs/ext/skills/Cargo.toml
@@ -48,5 +48,5 @@ insta = { workspace = true }
opentelemetry_sdk = { workspace = true }
pretty_assertions = { workspace = true }
tempfile = { workspace = true }
-tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }
+tokio = { workspace = true, features = ["macros", "rt-multi-thread", "test-util"] }
tracing-subscriber = { workspace = true }
diff --git a/codex-rs/ext/skills/src/catalog_state.rs b/codex-rs/ext/skills/src/catalog_state.rs
index 9f8487e40a..d0fe4c8d13 100644
--- a/codex-rs/ext/skills/src/catalog_state.rs
+++ b/codex-rs/ext/skills/src/catalog_state.rs
@@ -7,6 +7,7 @@ use std::sync::Arc;
use codex_exec_server::ExecutorCapabilityDiscoverySnapshot;
use codex_exec_server::ResolvedSelectedCapabilityRoot;
+use codex_extension_api::CatalogRefreshBackoff;
use codex_mcp::McpResourceClient;
use codex_mcp::McpResourceClientAuthKey;
use codex_mcp::McpResourceServerCacheKey;
@@ -33,6 +34,7 @@ pub(crate) struct SkillCatalogState {
pub(crate) executor_v2_cache: Option<CachedExecutorV2Catalog>,
// Shared by both discovery modes.
pub(crate) cloud_generation: Option<Arc<CloudSkillGeneration>>,
+ pub(crate) cloud_retry: Option<CatalogRefreshBackoff>,
pub(crate) cloud_resources: CloudResourceCache,
}
diff --git a/codex-rs/ext/skills/src/cloud_cache_tests.rs b/codex-rs/ext/skills/src/cloud_cache_tests.rs
index a85ed10aab..4dd3b7fd34 100644
--- a/codex-rs/ext/skills/src/cloud_cache_tests.rs
+++ b/codex-rs/ext/skills/src/cloud_cache_tests.rs
@@ -19,6 +19,7 @@ use pretty_assertions::assert_eq;
use std::sync::atomic::AtomicBool;
use std::sync::atomic::AtomicUsize;
use std::sync::atomic::Ordering;
+use std::time::Duration;
use tokio::sync::Notify;
#[derive(Default)]
@@ -204,7 +205,7 @@ async fn frozen_cloud_catalog_and_reads_ignore_live_mcp_changes() {
}
}
-#[tokio::test]
+#[tokio::test(start_paused = true)]
async fn step_cloud_refresh_serializes_waiters_and_retries() {
#[derive(Default)]
struct WarningCounter(AtomicUsize);
@@ -243,13 +244,16 @@ async fn step_cloud_refresh_serializes_waiters_and_retries() {
turn_store: &turn_store,
};
- // Retry every failure, but warn only once during the turn.
- for expected_calls in 1..=3 {
+ for (index, delay) in [30, 60, 120, 240, 300, 300].into_iter().enumerate() {
cloud.fail_list.store(true, Ordering::SeqCst);
assert!(!contributor.refresh_catalog_for_step(refresh_input).await);
assert_eq!(thread_state.available_cloud_catalog(), None);
- assert_eq!(cloud.lists.load(Ordering::SeqCst), expected_calls);
+ assert_eq!(cloud.lists.load(Ordering::SeqCst), index + 1);
assert_eq!(warnings.0.load(Ordering::SeqCst), 1);
+ tokio::time::advance(Duration::from_secs(delay - 1)).await;
+ assert!(!contributor.refresh_catalog_for_step(refresh_input).await);
+ assert_eq!(cloud.lists.load(Ordering::SeqCst), index + 1);
+ tokio::time::advance(Duration::from_secs(1)).await;
}
cloud.pause_list.store(true, Ordering::SeqCst);
let mut cancelled = contributor.refresh_catalog_for_step(refresh_input);
@@ -262,9 +266,9 @@ async fn step_cloud_refresh_serializes_waiters_and_retries() {
Some(SkillCatalog::default());
assert!(!contributor.refresh_catalog_for_step(refresh_input).await);
assert!(!contributor.refresh_catalog_for_step(refresh_input).await);
- assert_eq!(cloud.lists.load(Ordering::SeqCst), 5);
+ assert_eq!(cloud.lists.load(Ordering::SeqCst), 8);
- // Waiting callers reuse success, but retry a failed or cancelled discovery.
+ // Waiting callers reuse success and share failures' cooldowns; cancellation allows retry.
for (fails, cancelled) in [(false, false), (true, false), (false, true)] {
session_store.insert(SkillsSessionState {
mcp_resources: Some(client()),
@@ -312,6 +316,12 @@ async fn step_cloud_refresh_serializes_waiters_and_retries() {
.await
.expect("waiting discovery should complete after its predecessor stops")
);
+ if fails {
+ assert_eq!(thread_state.available_cloud_catalog(), None);
+ assert_eq!(cloud.lists.load(Ordering::SeqCst), previous_calls + 1);
+ tokio::time::advance(Duration::from_secs(30)).await;
+ assert!(!contributor.refresh_catalog_for_step(prewarm_input).await);
+ }
assert_eq!(
thread_state.available_cloud_catalog(),
Some(if fails || cancelled {
@@ -329,22 +339,40 @@ async fn step_cloud_refresh_serializes_waiters_and_retries() {
assert_eq!(warnings.0.load(Ordering::SeqCst), 1);
}
- // A fresh turn may warn again while discovery remains unavailable.
+ // New turns share the thread's cooldown and failure progression.
session_store.insert(SkillsSessionState {
mcp_resources: Some(client()),
extension_metrics: None,
});
- cloud.fail_list.store(true, Ordering::SeqCst);
let next_turn = ExtensionData::new("next turn");
- assert!(
- !contributor
- .refresh_catalog_for_step(CatalogRefreshInput {
- turn_id: "next turn",
- turn_store: &next_turn,
- ..refresh_input
- })
- .await
- );
+ let previous_calls = cloud.lists.load(Ordering::SeqCst);
+ for turn_store in [&turn_store, &next_turn] {
+ cloud.fail_list.store(true, Ordering::SeqCst);
+ assert!(
+ !contributor
+ .refresh_catalog_for_step(CatalogRefreshInput {
+ turn_store,
+ ..refresh_input
+ })
+ .await
+ );
+ assert_eq!(cloud.lists.load(Ordering::SeqCst), previous_calls + 1);
+ }
+ assert_eq!(warnings.0.load(Ordering::SeqCst), 1);
+ let next_input = CatalogRefreshInput {
+ turn_id: "next turn",
+ turn_store: &next_turn,
+ ..refresh_input
+ };
+ tokio::time::advance(Duration::from_secs(30)).await;
+ assert!(!contributor.refresh_catalog_for_step(next_input).await);
+ assert_eq!(cloud.lists.load(Ordering::SeqCst), previous_calls + 2);
+ tokio::time::advance(Duration::from_secs(59)).await;
+ assert!(!contributor.refresh_catalog_for_step(refresh_input).await);
+ assert_eq!(cloud.lists.load(Ordering::SeqCst), previous_calls + 2);
+ tokio::time::advance(Duration::from_secs(1)).await;
+ assert!(!contributor.refresh_catalog_for_step(next_input).await);
+ assert_eq!(cloud.lists.load(Ordering::SeqCst), previous_calls + 3);
assert_eq!(warnings.0.load(Ordering::SeqCst), 2);
}
@@ -416,7 +444,7 @@ async fn late_discovery_cannot_publish_into_a_new_generation() {
);
}
-#[tokio::test]
+#[tokio::test(start_paused = true)]
async fn partial_cloud_catalog_without_warnings_is_refreshed_next_turn() {
let state = state();
let provider = Arc::new(Provider::default());
@@ -472,6 +500,7 @@ async fn partial_cloud_catalog_without_warnings_is_refreshed_next_turn() {
);
assert!(state.available_cloud_catalog().is_some());
assert!(retry_turn.get::<LatestCloudSkillsAuth>().is_none());
+ tokio::time::advance(Duration::from_secs(30)).await;
state
.refresh_cloud_catalog(&providers, query(client), &retry_turn)
.await
diff --git a/codex-rs/ext/skills/src/state.rs b/codex-rs/ext/skills/src/state.rs
index 9bed213b4f..20c505d8fe 100644
--- a/codex-rs/ext/skills/src/state.rs
+++ b/codex-rs/ext/skills/src/state.rs
@@ -7,6 +7,7 @@ use std::sync::Mutex;
use tokio::sync::Mutex as AsyncMutex;
use codex_exec_server::EnvironmentAccessKey;
+use codex_extension_api::CatalogRefreshBackoff;
use codex_extension_api::ExtensionData;
use codex_extension_api::ExtensionMetrics;
use codex_mcp::McpResourceClient;
@@ -97,6 +98,7 @@ impl SkillsThreadState {
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
catalogs.cloud_generation = None;
+ catalogs.cloud_retry = None;
catalogs.cloud_resources = CloudResourceCache::default();
}
*previous = config;
@@ -263,7 +265,7 @@ impl SkillsThreadState {
.and_then(|cache| cache.catalog.clone())
}
- /// Reuses successful discovery within a turn; retries failures next step.
+ /// Reuses discovery within a turn; backs off failures.
#[expect(
clippy::await_holding_invalid_type,
reason = "cloud discovery must remain serialized through fetching and catalog publication"
@@ -304,8 +306,13 @@ impl SkillsThreadState {
{
return Ok(());
}
- turn_store.remove::<LatestCloudSkillsAuth>();
let resource_cache_key = cache.current_resource_cache_key();
+ if catalogs.cloud_retry.as_ref().is_some_and(|retry| {
+ retry.is_pending(cache.auth_cache_key.as_ref(), resource_cache_key.as_ref())
+ }) {
+ return Ok(());
+ }
+ turn_store.remove::<LatestCloudSkillsAuth>();
if cache.is_auth_current()
&& cache.catalog.as_ref().is_some_and(|catalog| {
(waited && *refresh_succeeded)
@@ -313,10 +320,11 @@ impl SkillsThreadState {
})
&& cache.resource_cache_key == resource_cache_key
{
+ catalogs.cloud_retry = None;
turn_store.insert(LatestCloudSkillsAuth(cache.auth_cache_key.clone()));
return Ok(());
}
- // A failed or cancelled refresh must leave waiting callers able to retry.
+ // Waiting callers must not reuse a failed or cancelled refresh.
*refresh_succeeded = false;
(cache, resource_cache_key)
};
@@ -338,7 +346,18 @@ impl SkillsThreadState {
// Ignore superseded results, including errors.
return Ok(());
}
- let mut catalog = result?;
+ let mut catalog = match result {
+ Ok(catalog) => catalog,
+ Err(error) => {
+ catalogs.cloud_retry = Some(CatalogRefreshBackoff::after_failure(
+ cache.auth_cache_key.clone(),
+ resource_cache_key,
+ catalogs.cloud_retry.as_ref(),
+ ));
+ return Err(error);
+ }
+ };
+ catalogs.cloud_retry = None;
if catalog.is_partial
&& let Some(previous) = cache
.catalog