Skip to content

armgit / codex

Public Imported from GitHub
0

Lightweight coding agent that runs in your terminal

Rust

Keep discovered Apps tools available during startup recovery (#53672)

jif4c1cd8bbccb3
diff --git a/codex-rs/codex-mcp/src/connection_manager.rs b/codex-rs/codex-mcp/src/connection_manager.rs
index 7aaba94fbc..0f70d060bd 100644
--- a/codex-rs/codex-mcp/src/connection_manager.rs
+++ b/codex-rs/codex-mcp/src/connection_manager.rs
@@ -138,6 +138,21 @@ impl McpServerConnection {
self.client.client().await
}
+ async fn client_for_call(&self) -> Result<ManagedClient, StartupOutcomeError> {
+ let result = self.client().await;
+ if matches!(result, Err(StartupOutcomeError::Failed { .. }))
+ && !self.client.cancel_token.is_cancelled()
+ && let Some(reconnect) = &self.client.startup_reconnect
+ {
+ if let Some(attempt) = reconnect.reconnect_in_background() {
+ return attempt.await;
+ }
+ // Recovery may have finished between reading the client and joining it.
+ return self.client().await;
+ }
+ result
+ }
+
async fn shutdown(&self) {
self.client.shutdown().await;
}
@@ -924,7 +939,8 @@ impl McpConnectionSet {
let Some(view) = self.servers.get(server_name) else {
return false;
};
- view.connection.client.ready_transport().is_some() || view.connection.client().await.is_ok()
+ view.connection.client.ready_transport().is_some()
+ || view.connection.client_for_call().await.is_ok()
}
/// Stop all MCP clients owned by this manager and terminate stdio server processes.
diff --git a/codex-rs/codex-mcp/src/connection_manager/tool_catalog.rs b/codex-rs/codex-mcp/src/connection_manager/tool_catalog.rs
index 7ebf5af6f7..6aeae1ca2e 100644
--- a/codex-rs/codex-mcp/src/connection_manager/tool_catalog.rs
+++ b/codex-rs/codex-mcp/src/connection_manager/tool_catalog.rs
@@ -364,7 +364,12 @@ impl McpConnectionSet {
.client
.startup_complete
.load(Ordering::Acquire);
- let accepts_cached_catalog = view.startup_readiness == McpStartupReadiness::Catalog;
+ // A transient Apps startup failure must not remove functions already discovered
+ // by the model. Keep retrying even when the cached catalog supplies this step.
+ view.connection.client.reconnect_failed_startup().await;
+ let accepts_cached_catalog = view.startup_readiness == McpStartupReadiness::Catalog
+ || (view.connection.client.is_codex_apps_mcp_server
+ && !view.connection.client.cancel_token.is_cancelled());
let cached_tools = if startup_pending
|| (accepts_cached_catalog && view.connection.client.ready_client().is_none())
{
@@ -383,7 +388,6 @@ impl McpConnectionSet {
{
return None;
}
- view.connection.client.reconnect_failed_startup().await;
let Ok(mut client) = view.connection.client().await else {
trace!(server_name = %server_name, "omitting MCP server without an exact ready client");
return None;
@@ -544,6 +548,7 @@ impl McpConnectionSet {
if !view.tool_filter.allows(&advertised_tool.tool.name) {
return None;
}
+ // Recovery belongs to readiness, before the caller acquires the tool execution lock.
let mut client = view.connection.client().await.ok()?;
client.tool_timeout = view.tool_timeout;
let snapshot = client.tool_catalog.read(Arc::new).await;
diff --git a/codex-rs/codex-mcp/src/connection_manager_tests.rs b/codex-rs/codex-mcp/src/connection_manager_tests.rs
index d7a841bf4c..4e427d3a21 100644
--- a/codex-rs/codex-mcp/src/connection_manager_tests.rs
+++ b/codex-rs/codex-mcp/src/connection_manager_tests.rs
@@ -4182,13 +4182,13 @@ async fn later_tool_list_retries_after_failed_reconnect_and_keeps_cached_tools()
.boxed()
.shared()
});
- let manager = create_test_manager_with_failed_apps_startup(
+ let manager = Arc::new(create_test_manager_with_failed_apps_startup(
vec![create_test_tool(
CODEX_APPS_MCP_SERVER_NAME,
"cached_drive_search",
)],
reconnect_factory,
- );
+ ));
let first_reconnect_finished = reconnect_finished.notified();
let tools = manager.list_all_tools().await;
@@ -4202,6 +4202,19 @@ async fn later_tool_list_retries_after_failed_reconnect_and_keeps_cached_tools()
first_reconnect_finished.await;
assert_eq!(attempts.load(std::sync::atomic::Ordering::SeqCst), 1);
+ let failed_binding = capture_binding(&manager).await;
+ assert_eq!(failed_binding.tools(), tools);
+ assert!(
+ manager
+ .prepare_call_for_tool(
+ Arc::clone(failed_binding.config()),
+ &failed_binding.tools()[0],
+ )
+ .await
+ .is_none()
+ );
+ assert_eq!(attempts.load(Ordering::SeqCst), 1);
+
let tools = manager.list_all_tools().await;
assert_eq!(
tools
@@ -4287,10 +4300,7 @@ async fn tool_lists_do_not_block_and_share_codex_apps_startup_reconnect() {
.shared()
});
let mut manager = create_test_manager_with_failed_apps_startup(
- vec![create_test_tool(
- CODEX_APPS_MCP_SERVER_NAME,
- "cached_drive_search",
- )],
+ vec![create_test_tool(CODEX_APPS_MCP_SERVER_NAME, "drive_search")],
reconnect_factory,
);
manager.set_test_server_metadata(
@@ -4320,25 +4330,32 @@ async fn tool_lists_do_not_block_and_share_codex_apps_startup_reconnect() {
.iter()
.map(|tool| tool.callable_name.as_str())
.collect::<Vec<_>>(),
- vec!["cached_drive_search"]
+ vec!["drive_search"]
);
assert_eq!(
second_tools
.iter()
.map(|tool| tool.callable_name.as_str())
.collect::<Vec<_>>(),
- vec!["cached_drive_search"]
+ vec!["drive_search"]
);
let pending_step = tokio::time::timeout(Duration::from_millis(10), capture_binding(&manager))
.await
.expect("step capture should not wait for reconnect");
- assert!(
- pending_step.tools().is_empty(),
- "a model step must not advertise cached tools without an exact ready client"
- );
+ assert_eq!(pending_step.tools(), first_tools);
+ // Readiness joins the retry that discovery started before call preparation.
+ let mut ready = Box::pin(manager.wait_for_server_startup(CODEX_APPS_MCP_SERVER_NAME));
+ assert!(futures::poll!(&mut ready).is_pending());
+ assert_eq!(attempts.load(Ordering::SeqCst), 1);
release_reconnect.notify_one();
- tokio::task::yield_now().await;
+ assert!(ready.await);
+ assert!(
+ manager
+ .prepare_call_for_tool(Arc::clone(pending_step.config()), &pending_step.tools()[0])
+ .await
+ .is_some()
+ );
let tools = manager.list_all_tools().await;
assert_eq!(
tools
diff --git a/codex-rs/codex-mcp/src/rmcp_client.rs b/codex-rs/codex-mcp/src/rmcp_client.rs
index 8e3a8d26b1..e73030d036 100644
--- a/codex-rs/codex-mcp/src/rmcp_client.rs
+++ b/codex-rs/codex-mcp/src/rmcp_client.rs
@@ -4,6 +4,7 @@
//! initializing the server, listing raw tools, applying per-server tool filters,
//! and exposing cached Codex Apps tools while a client is still connecting.
//! Initialization capabilities survive tool-discovery failures and reset on a new attempt.
+//! Apps discovery stays nonblocking; invocations can join the shared startup retry.
//! Executor-discovered environment credentials remain executor-bound across those attempts.
//! Higher-level aggregation and resource/tool APIs live in
//! [`crate::connection_manager`].
@@ -162,7 +163,7 @@ pub(crate) type ManagedClientFuture =
struct CodexAppsStartupReconnectState {
current_client: Option<ManagedClient>,
last_error: Option<StartupOutcomeError>,
- reconnect_in_flight: bool,
+ reconnect_in_flight: Option<ManagedClientFuture>,
consecutive_failures: u32,
retry_not_before: Option<TokioInstant>,
}
@@ -211,26 +212,24 @@ impl CodexAppsStartupReconnect {
.clone()
}
- fn reconnect_in_background(self: &Arc<Self>) {
- {
- let mut state = self
- .state
- .lock()
- .unwrap_or_else(std::sync::PoisonError::into_inner);
- if state.current_client.is_some() || state.reconnect_in_flight {
- return;
- }
- if state
+ pub(crate) fn reconnect_in_background(self: &Arc<Self>) -> Option<ManagedClientFuture> {
+ let mut state = self
+ .state
+ .lock()
+ .unwrap_or_else(std::sync::PoisonError::into_inner);
+ if let Some(attempt) = &state.reconnect_in_flight {
+ return Some(attempt.clone());
+ }
+ if state.current_client.is_some()
+ || state
.retry_not_before
.is_some_and(|retry_not_before| TokioInstant::now() < retry_not_before)
- {
- return;
- }
- state.reconnect_in_flight = true;
+ {
+ return None;
}
let reconnect = Arc::clone(self);
- tokio::spawn(async move {
+ let attempt = async move {
let result = (reconnect.factory)().await;
let startup_status_context = reconnect.startup_status_context.clone();
let recovered = {
@@ -238,10 +237,10 @@ impl CodexAppsStartupReconnect {
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
- state.reconnect_in_flight = false;
- match result {
+ state.reconnect_in_flight = None;
+ match &result {
Ok(client) => {
- state.current_client = Some(client);
+ state.current_client = Some(client.clone());
state.last_error = None;
state.consecutive_failures = 0;
state.retry_not_before = None;
@@ -274,7 +273,15 @@ impl CodexAppsStartupReconnect {
})
.await;
}
- });
+ result
+ }
+ .boxed()
+ .shared();
+ state.reconnect_in_flight = Some(attempt.clone());
+ drop(state);
+ // Keep recovery running if the invocation waiting for it is cancelled.
+ tokio::spawn(attempt.clone());
+ Some(attempt)
}
}
@@ -569,7 +576,7 @@ impl AsyncManagedClient {
return;
}
if matches!(self.client().await, Err(StartupOutcomeError::Failed { .. })) {
- startup_reconnect.reconnect_in_background();
+ let _ = startup_reconnect.reconnect_in_background();
}
}
diff --git a/codex-rs/codex-mcp/src/rmcp_client/status.rs b/codex-rs/codex-mcp/src/rmcp_client/status.rs
index b166743c77..6bb9af07a3 100644
--- a/codex-rs/codex-mcp/src/rmcp_client/status.rs
+++ b/codex-rs/codex-mcp/src/rmcp_client/status.rs
@@ -15,7 +15,7 @@ impl AsyncManagedClient {
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
- if state.reconnect_in_flight {
+ if state.reconnect_in_flight.is_some() {
return Status::Starting;
}
state
diff --git a/codex-rs/core/tests/common/apps_test_server.rs b/codex-rs/core/tests/common/apps_test_server.rs
index 4ae5d8034a..cde2ed74da 100644
--- a/codex-rs/core/tests/common/apps_test_server.rs
+++ b/codex-rs/core/tests/common/apps_test_server.rs
@@ -11,6 +11,7 @@ use codex_login::CodexAuth;
use codex_models_manager::bundled_models_response;
use serde_json::Value;
use serde_json::json;
+use std::collections::VecDeque;
use std::sync::Arc;
use std::sync::Mutex;
use std::sync::atomic::AtomicBool;
@@ -60,7 +61,7 @@ const CALENDAR_LIST_EVENTS_RESOURCE_URI: &str = "connector://calendar/tools/cale
pub const DOCUMENT_EXTRACT_TEXT_RESOURCE_URI: &str =
"connector://calendar/tools/calendar_extract_text";
-type AppsStartupInitializeGate = Arc<Mutex<Option<mpsc::Receiver<()>>>>;
+type AppsStartupInitializeGate = Arc<Mutex<VecDeque<mpsc::Receiver<()>>>>;
pub struct HostedMessagingServer(pub String);
@@ -112,12 +113,13 @@ impl AppsTestServerStartupControl {
}
/// Holds one successful startup without turning recovery into a fresh runtime refresh.
+ /// Multiple holds are consumed in order, so tests can gate startup and recovery in advance.
pub fn hold_next_successful_initialize(&self) -> mpsc::Sender<()> {
let (release, gate) = mpsc::channel();
- *self
- .successful_initialize_gate
+ self.successful_initialize_gate
.lock()
- .expect("Apps initialization gate lock should not be poisoned") = Some(gate);
+ .expect("Apps initialization gate lock should not be poisoned")
+ .push_back(gate);
release
}
}
@@ -224,7 +226,7 @@ impl AppsTestServer {
let control = AppsTestServerStartupControl {
initialize_attempts: Arc::new(AtomicUsize::new(0)),
remaining_initialize_failures: Arc::new(AtomicUsize::new(0)),
- successful_initialize_gate: Arc::new(Mutex::new(None)),
+ successful_initialize_gate: Arc::new(Mutex::new(VecDeque::new())),
};
mount_streamable_http_json_rpc_with_startup_control(
server,
@@ -554,7 +556,7 @@ impl Respond for CodexAppsJsonRpcResponder {
let gate = self.successful_initialize_gate.as_ref().and_then(|gate| {
gate.lock()
.expect("Apps initialization gate lock should not be poisoned")
- .take()
+ .pop_front()
});
// This responder's dedicated mock server stays paused while the test controls
// when this existing client's background retry may finish.
diff --git a/codex-rs/core/tests/suite/mcp_tool_exposure.rs b/codex-rs/core/tests/suite/mcp_tool_exposure.rs
index 67440412c0..8e557c4eaa 100644
--- a/codex-rs/core/tests/suite/mcp_tool_exposure.rs
+++ b/codex-rs/core/tests/suite/mcp_tool_exposure.rs
@@ -40,6 +40,7 @@ use core_test_support::responses;
use core_test_support::responses::ResponsesRequest;
use core_test_support::responses::ev_assistant_message;
use core_test_support::responses::ev_completed;
+use core_test_support::responses::ev_custom_tool_call;
use core_test_support::responses::ev_function_call;
use core_test_support::responses::ev_function_call_with_namespace;
use core_test_support::responses::ev_response_created;
@@ -1529,6 +1530,230 @@ async fn apps_guidance_and_deferred_namespace_appear_after_recovery_within_a_tur
Ok(())
}
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn code_mode_discovered_apps_tool_survives_startup_failure_and_recovers() -> Result<()> {
+ skip_if_no_network!(Ok(()));
+
+ let server = responses::start_mock_server().await;
+ // Holding MCP initialization must not block inference or connector directory requests.
+ let mcp_server = responses::start_mock_server().await;
+ let (gated_apps, startup) = AppsTestServer::mount_with_startup_control(&mcp_server).await?;
+ let apps = AppsTestServer::mount_searchable(&server).await?;
+ let mut extensions = ExtensionRegistryBuilder::new();
+ extensions.mcp_server_contributor(Arc::new(AppsMcpServerContributor {
+ id: "hosted_plugin_runtime",
+ url: format!("{}/api/codex/ps/mcp", gated_apps.chatgpt_base_url),
+ root_resolved: None,
+ }));
+ let extensions = Arc::new(extensions.build());
+ let code_mode_host = codex_utils_cargo_bin::cargo_bin("codex-code-mode-host")?;
+ let builder = || {
+ search_capable_apps_builder(apps.chatgpt_base_url.clone())
+ .with_extensions(Arc::clone(&extensions))
+ .with_code_mode_host_program(code_mode_host.clone())
+ .with_model_info_override("gpt-5.5", |model| {
+ model.tool_mode = Some(codex_protocol::openai_models::ToolMode::CodeMode);
+ })
+ .with_config(|config| {
+ config.permissions.approval_policy = Constrained::allow_any(AskForApproval::Never);
+ config
+ .features
+ .enable(Feature::CodeMode)
+ .expect("test config should allow Code Mode");
+ config
+ .features
+ .enable(Feature::CodeModeHost)
+ .expect("test config should allow the Code Mode host");
+ config
+ .features
+ .enable(Feature::DefaultModeRequestUserInput)
+ .expect("test config should allow user input");
+ })
+ };
+ let warm = builder().build_with_auto_env(&server).await?;
+ wait_for_mcp_server(&warm.codex, CODEX_APPS_MCP_SERVER_NAME).await?;
+ warm.codex.ensure_rollout_materialized().await;
+
+ // Restart with the persisted catalog and hold discovery until the model has used it.
+ // Queue recovery too: startup schedules its retry immediately after reporting failure.
+ let initial_startup = startup.hold_next_successful_initialize();
+ let recovery = startup.hold_next_successful_initialize();
+ let test = builder().restart_with_auto_env(&server, &warm).await?;
+ let question = json!({
+ "questions": [{
+ "id": "continue",
+ "header": "Continue",
+ "question": "Continue the tool call?",
+ "options": [
+ {"label": "Yes (Recommended)", "description": "Continue."},
+ {"label": "No", "description": "Stop."}
+ ]
+ }]
+ });
+ let response = mount_sse_sequence(
+ &server,
+ vec![
+ sse(vec![
+ ev_response_created("discover"),
+ ev_custom_tool_call(
+ "discover",
+ "exec",
+ "const tool = ALL_TOOLS.find(t => t.name.endsWith('list_events'));\n\
+ store('calendar-tool', tool.name); text(tool.name);",
+ ),
+ ev_completed("discover"),
+ ]),
+ sse(vec![
+ ev_response_created("after-discovery"),
+ ev_function_call(
+ "after-discovery",
+ "request_user_input",
+ &question.to_string(),
+ ),
+ ev_completed("after-discovery"),
+ ]),
+ sse(vec![
+ ev_response_created("invoke"),
+ ev_custom_tool_call(
+ "invoke",
+ "exec",
+ "const pending = tools[load('calendar-tool')]({});\n\
+ yield_control();\n\
+ text(await pending);",
+ ),
+ ev_completed("invoke"),
+ ]),
+ sse(vec![
+ ev_response_created("after-invocation"),
+ ev_function_call(
+ "after-invocation",
+ "request_user_input",
+ &question.to_string(),
+ ),
+ ev_completed("after-invocation"),
+ ]),
+ ],
+ )
+ .await;
+ test.codex
+ .start_or_steer_turn(TurnInputRequest::user_input(vec![UserInput::Text {
+ text: "Discover Calendar, then list events.".into(),
+ text_elements: Vec::new(),
+ }]))
+ .await?;
+ let EventMsg::RequestUserInput(discovered) = wait_for_event(&test.codex, |event| {
+ assert!(
+ !matches!(event, EventMsg::TurnComplete(_)),
+ "turn ended before discovering the cached tool"
+ );
+ matches!(event, EventMsg::RequestUserInput(_))
+ })
+ .await
+ else {
+ unreachable!()
+ };
+
+ // Dropping the held response fails it; fail the transport retry as well.
+ startup.fail_next_initialize_attempts(/*attempts*/ 1);
+ drop(initial_startup);
+ wait_for_event(&test.codex, |event| {
+ matches!(
+ event, EventMsg::McpStartupUpdate(update)
+ if update.server == CODEX_APPS_MCP_SERVER_NAME
+ && matches!(update.status, codex_protocol::protocol::McpStartupStatus::Failed { .. })
+ )
+ })
+ .await;
+ test.codex
+ .submit(Op::UserInputAnswer {
+ id: discovered.turn_id,
+ response: RequestUserInputResponse {
+ answers: HashMap::from([(
+ "continue".to_string(),
+ RequestUserInputAnswer {
+ answers: vec!["Yes (Recommended)".to_string()],
+ },
+ )]),
+ },
+ })
+ .await?;
+
+ // This prompt is after the invocation in the second cell: the function must still exist,
+ // and waiting for its connection must leave unrelated tools able to execute.
+ let EventMsg::RequestUserInput(invoked) = wait_for_event(&test.codex, |event| {
+ assert!(
+ !matches!(event, EventMsg::TurnComplete(_)),
+ "turn ended before invoking the cached tool"
+ );
+ matches!(event, EventMsg::RequestUserInput(_))
+ })
+ .await
+ else {
+ unreachable!()
+ };
+ let requests = response.requests();
+ assert_eq!(requests.len(), 4);
+ assert!(
+ super::code_mode::custom_tool_output_last_non_empty_text(&requests[1], "discover")
+ .expect("discovery should return the cached tool name")
+ .ends_with("list_events")
+ );
+ let running = super::code_mode::custom_tool_output_last_non_empty_text(&requests[3], "invoke")
+ .expect("the invocation should wait for Apps recovery");
+ let cell_id = running
+ .strip_prefix("Script running with cell ID ")
+ .and_then(|text| text.lines().next())
+ .expect("invoking the discovered function should yield, not fail");
+ let finished = responses::mount_function_call_agent_response(
+ &server,
+ "finish",
+ &json!({"cell_id": cell_id, "yield_time_ms": 10_000}).to_string(),
+ "wait",
+ )
+ .await;
+ recovery
+ .send(())
+ .expect("the shared Apps retry should still be waiting");
+ test.codex
+ .submit(Op::UserInputAnswer {
+ id: invoked.turn_id,
+ response: RequestUserInputResponse {
+ answers: HashMap::from([(
+ "continue".to_string(),
+ RequestUserInputAnswer {
+ answers: vec!["Yes (Recommended)".to_string()],
+ },
+ )]),
+ },
+ })
+ .await?;
+ wait_for_event(&test.codex, |event| {
+ matches!(event, EventMsg::TurnComplete(_))
+ })
+ .await;
+
+ let output = finished
+ .completion
+ .single_request()
+ .function_call_output("finish");
+ assert!(
+ output["output"]
+ .as_array()
+ .expect("wait should return Code Mode content items")
+ .iter()
+ .any(|item| item["text"]
+ .as_str()
+ .is_some_and(|text| text.contains("called calendar_list_events"))),
+ "{output}"
+ );
+ let calls = core_test_support::apps_test_server::recorded_apps_tool_calls(&mcp_server).await;
+ assert_eq!(calls.len(), 1);
+ assert_eq!(calls[0]["params"]["name"], "calendar_list_events");
+ assert_eq!(startup.initialize_attempts(), 4);
+ test.codex.shutdown_and_wait().await?;
+ Ok(())
+}
+
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn later_follow_up_uses_background_recovered_apps_after_mid_thread_startup_failures()
-> Result<()> {