Lightweight coding agent that runs in your terminal
Rust
Keep discovered Apps tools available during startup recovery (#53672)
jif4c1cd8bbccb3
diff --git a/codex-rs/codex-mcp/src/connection_manager.rs b/codex-rs/codex-mcp/src/connection_manager.rsindex 7aaba94fbc..0f70d060bd 100644--- a/codex-rs/codex-mcp/src/connection_manager.rs+++ b/codex-rs/codex-mcp/src/connection_manager.rs@@ -138,6 +138,21 @@ impl McpServerConnection {self.client.client().await}+ async fn client_for_call(&self) -> Result<ManagedClient, StartupOutcomeError> {+ let result = self.client().await;+ if matches!(result, Err(StartupOutcomeError::Failed { .. }))+ && !self.client.cancel_token.is_cancelled()+ && let Some(reconnect) = &self.client.startup_reconnect+ {+ if let Some(attempt) = reconnect.reconnect_in_background() {+ return attempt.await;+ }+ // Recovery may have finished between reading the client and joining it.+ return self.client().await;+ }+ result+ }+async fn shutdown(&self) {self.client.shutdown().await;}@@ -924,7 +939,8 @@ impl McpConnectionSet {let Some(view) = self.servers.get(server_name) else {return false;};- view.connection.client.ready_transport().is_some() || view.connection.client().await.is_ok()+ view.connection.client.ready_transport().is_some()+ || view.connection.client_for_call().await.is_ok()}/// Stop all MCP clients owned by this manager and terminate stdio server processes.diff --git a/codex-rs/codex-mcp/src/connection_manager/tool_catalog.rs b/codex-rs/codex-mcp/src/connection_manager/tool_catalog.rsindex 7ebf5af6f7..6aeae1ca2e 100644--- a/codex-rs/codex-mcp/src/connection_manager/tool_catalog.rs+++ b/codex-rs/codex-mcp/src/connection_manager/tool_catalog.rs@@ -364,7 +364,12 @@ impl McpConnectionSet {.client.startup_complete.load(Ordering::Acquire);- let accepts_cached_catalog = view.startup_readiness == McpStartupReadiness::Catalog;+ // A transient Apps startup failure must not remove functions already discovered+ // by the model. Keep retrying even when the cached catalog supplies this step.+ view.connection.client.reconnect_failed_startup().await;+ let accepts_cached_catalog = view.startup_readiness == McpStartupReadiness::Catalog+ || (view.connection.client.is_codex_apps_mcp_server+ && !view.connection.client.cancel_token.is_cancelled());let cached_tools = if startup_pending|| (accepts_cached_catalog && view.connection.client.ready_client().is_none()){@@ -383,7 +388,6 @@ impl McpConnectionSet {{return None;}- view.connection.client.reconnect_failed_startup().await;let Ok(mut client) = view.connection.client().await else {trace!(server_name = %server_name, "omitting MCP server without an exact ready client");return None;@@ -544,6 +548,7 @@ impl McpConnectionSet {if !view.tool_filter.allows(&advertised_tool.tool.name) {return None;}+ // Recovery belongs to readiness, before the caller acquires the tool execution lock.let mut client = view.connection.client().await.ok()?;client.tool_timeout = view.tool_timeout;let snapshot = client.tool_catalog.read(Arc::new).await;diff --git a/codex-rs/codex-mcp/src/connection_manager_tests.rs b/codex-rs/codex-mcp/src/connection_manager_tests.rsindex d7a841bf4c..4e427d3a21 100644--- a/codex-rs/codex-mcp/src/connection_manager_tests.rs+++ b/codex-rs/codex-mcp/src/connection_manager_tests.rs@@ -4182,13 +4182,13 @@ async fn later_tool_list_retries_after_failed_reconnect_and_keeps_cached_tools().boxed().shared()});- let manager = create_test_manager_with_failed_apps_startup(+ let manager = Arc::new(create_test_manager_with_failed_apps_startup(vec![create_test_tool(CODEX_APPS_MCP_SERVER_NAME,"cached_drive_search",)],reconnect_factory,- );+ ));let first_reconnect_finished = reconnect_finished.notified();let tools = manager.list_all_tools().await;@@ -4202,6 +4202,19 @@ async fn later_tool_list_retries_after_failed_reconnect_and_keeps_cached_tools()first_reconnect_finished.await;assert_eq!(attempts.load(std::sync::atomic::Ordering::SeqCst), 1);+ let failed_binding = capture_binding(&manager).await;+ assert_eq!(failed_binding.tools(), tools);+ assert!(+ manager+ .prepare_call_for_tool(+ Arc::clone(failed_binding.config()),+ &failed_binding.tools()[0],+ )+ .await+ .is_none()+ );+ assert_eq!(attempts.load(Ordering::SeqCst), 1);+let tools = manager.list_all_tools().await;assert_eq!(tools@@ -4287,10 +4300,7 @@ async fn tool_lists_do_not_block_and_share_codex_apps_startup_reconnect() {.shared()});let mut manager = create_test_manager_with_failed_apps_startup(- vec![create_test_tool(- CODEX_APPS_MCP_SERVER_NAME,- "cached_drive_search",- )],+ vec![create_test_tool(CODEX_APPS_MCP_SERVER_NAME, "drive_search")],reconnect_factory,);manager.set_test_server_metadata(@@ -4320,25 +4330,32 @@ async fn tool_lists_do_not_block_and_share_codex_apps_startup_reconnect() {.iter().map(|tool| tool.callable_name.as_str()).collect::<Vec<_>>(),- vec!["cached_drive_search"]+ vec!["drive_search"]);assert_eq!(second_tools.iter().map(|tool| tool.callable_name.as_str()).collect::<Vec<_>>(),- vec!["cached_drive_search"]+ vec!["drive_search"]);let pending_step = tokio::time::timeout(Duration::from_millis(10), capture_binding(&manager)).await.expect("step capture should not wait for reconnect");- assert!(- pending_step.tools().is_empty(),- "a model step must not advertise cached tools without an exact ready client"- );+ assert_eq!(pending_step.tools(), first_tools);+ // Readiness joins the retry that discovery started before call preparation.+ let mut ready = Box::pin(manager.wait_for_server_startup(CODEX_APPS_MCP_SERVER_NAME));+ assert!(futures::poll!(&mut ready).is_pending());+ assert_eq!(attempts.load(Ordering::SeqCst), 1);release_reconnect.notify_one();- tokio::task::yield_now().await;+ assert!(ready.await);+ assert!(+ manager+ .prepare_call_for_tool(Arc::clone(pending_step.config()), &pending_step.tools()[0])+ .await+ .is_some()+ );let tools = manager.list_all_tools().await;assert_eq!(toolsdiff --git a/codex-rs/codex-mcp/src/rmcp_client.rs b/codex-rs/codex-mcp/src/rmcp_client.rsindex 8e3a8d26b1..e73030d036 100644--- a/codex-rs/codex-mcp/src/rmcp_client.rs+++ b/codex-rs/codex-mcp/src/rmcp_client.rs@@ -4,6 +4,7 @@//! initializing the server, listing raw tools, applying per-server tool filters,//! and exposing cached Codex Apps tools while a client is still connecting.//! Initialization capabilities survive tool-discovery failures and reset on a new attempt.+//! Apps discovery stays nonblocking; invocations can join the shared startup retry.//! Executor-discovered environment credentials remain executor-bound across those attempts.//! Higher-level aggregation and resource/tool APIs live in//! [`crate::connection_manager`].@@ -162,7 +163,7 @@ pub(crate) type ManagedClientFuture =struct CodexAppsStartupReconnectState {current_client: Option<ManagedClient>,last_error: Option<StartupOutcomeError>,- reconnect_in_flight: bool,+ reconnect_in_flight: Option<ManagedClientFuture>,consecutive_failures: u32,retry_not_before: Option<TokioInstant>,}@@ -211,26 +212,24 @@ impl CodexAppsStartupReconnect {.clone()}- fn reconnect_in_background(self: &Arc<Self>) {- {- let mut state = self- .state- .lock()- .unwrap_or_else(std::sync::PoisonError::into_inner);- if state.current_client.is_some() || state.reconnect_in_flight {- return;- }- if state+ pub(crate) fn reconnect_in_background(self: &Arc<Self>) -> Option<ManagedClientFuture> {+ let mut state = self+ .state+ .lock()+ .unwrap_or_else(std::sync::PoisonError::into_inner);+ if let Some(attempt) = &state.reconnect_in_flight {+ return Some(attempt.clone());+ }+ if state.current_client.is_some()+ || state.retry_not_before.is_some_and(|retry_not_before| TokioInstant::now() < retry_not_before)- {- return;- }- state.reconnect_in_flight = true;+ {+ return None;}let reconnect = Arc::clone(self);- tokio::spawn(async move {+ let attempt = async move {let result = (reconnect.factory)().await;let startup_status_context = reconnect.startup_status_context.clone();let recovered = {@@ -238,10 +237,10 @@ impl CodexAppsStartupReconnect {.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner);- state.reconnect_in_flight = false;- match result {+ state.reconnect_in_flight = None;+ match &result {Ok(client) => {- state.current_client = Some(client);+ state.current_client = Some(client.clone());state.last_error = None;state.consecutive_failures = 0;state.retry_not_before = None;@@ -274,7 +273,15 @@ impl CodexAppsStartupReconnect {}).await;}- });+ result+ }+ .boxed()+ .shared();+ state.reconnect_in_flight = Some(attempt.clone());+ drop(state);+ // Keep recovery running if the invocation waiting for it is cancelled.+ tokio::spawn(attempt.clone());+ Some(attempt)}}@@ -569,7 +576,7 @@ impl AsyncManagedClient {return;}if matches!(self.client().await, Err(StartupOutcomeError::Failed { .. })) {- startup_reconnect.reconnect_in_background();+ let _ = startup_reconnect.reconnect_in_background();}}diff --git a/codex-rs/codex-mcp/src/rmcp_client/status.rs b/codex-rs/codex-mcp/src/rmcp_client/status.rsindex b166743c77..6bb9af07a3 100644--- a/codex-rs/codex-mcp/src/rmcp_client/status.rs+++ b/codex-rs/codex-mcp/src/rmcp_client/status.rs@@ -15,7 +15,7 @@ impl AsyncManagedClient {.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner);- if state.reconnect_in_flight {+ if state.reconnect_in_flight.is_some() {return Status::Starting;}statediff --git a/codex-rs/core/tests/common/apps_test_server.rs b/codex-rs/core/tests/common/apps_test_server.rsindex 4ae5d8034a..cde2ed74da 100644--- a/codex-rs/core/tests/common/apps_test_server.rs+++ b/codex-rs/core/tests/common/apps_test_server.rs@@ -11,6 +11,7 @@ use codex_login::CodexAuth;use codex_models_manager::bundled_models_response;use serde_json::Value;use serde_json::json;+use std::collections::VecDeque;use std::sync::Arc;use std::sync::Mutex;use std::sync::atomic::AtomicBool;@@ -60,7 +61,7 @@ const CALENDAR_LIST_EVENTS_RESOURCE_URI: &str = "connector://calendar/tools/calepub const DOCUMENT_EXTRACT_TEXT_RESOURCE_URI: &str ="connector://calendar/tools/calendar_extract_text";-type AppsStartupInitializeGate = Arc<Mutex<Option<mpsc::Receiver<()>>>>;+type AppsStartupInitializeGate = Arc<Mutex<VecDeque<mpsc::Receiver<()>>>>;pub struct HostedMessagingServer(pub String);@@ -112,12 +113,13 @@ impl AppsTestServerStartupControl {}/// Holds one successful startup without turning recovery into a fresh runtime refresh.+ /// Multiple holds are consumed in order, so tests can gate startup and recovery in advance.pub fn hold_next_successful_initialize(&self) -> mpsc::Sender<()> {let (release, gate) = mpsc::channel();- *self- .successful_initialize_gate+ self.successful_initialize_gate.lock()- .expect("Apps initialization gate lock should not be poisoned") = Some(gate);+ .expect("Apps initialization gate lock should not be poisoned")+ .push_back(gate);release}}@@ -224,7 +226,7 @@ impl AppsTestServer {let control = AppsTestServerStartupControl {initialize_attempts: Arc::new(AtomicUsize::new(0)),remaining_initialize_failures: Arc::new(AtomicUsize::new(0)),- successful_initialize_gate: Arc::new(Mutex::new(None)),+ successful_initialize_gate: Arc::new(Mutex::new(VecDeque::new())),};mount_streamable_http_json_rpc_with_startup_control(server,@@ -554,7 +556,7 @@ impl Respond for CodexAppsJsonRpcResponder {let gate = self.successful_initialize_gate.as_ref().and_then(|gate| {gate.lock().expect("Apps initialization gate lock should not be poisoned")- .take()+ .pop_front()});// This responder's dedicated mock server stays paused while the test controls// when this existing client's background retry may finish.diff --git a/codex-rs/core/tests/suite/mcp_tool_exposure.rs b/codex-rs/core/tests/suite/mcp_tool_exposure.rsindex 67440412c0..8e557c4eaa 100644--- a/codex-rs/core/tests/suite/mcp_tool_exposure.rs+++ b/codex-rs/core/tests/suite/mcp_tool_exposure.rs@@ -40,6 +40,7 @@ use core_test_support::responses;use core_test_support::responses::ResponsesRequest;use core_test_support::responses::ev_assistant_message;use core_test_support::responses::ev_completed;+use core_test_support::responses::ev_custom_tool_call;use core_test_support::responses::ev_function_call;use core_test_support::responses::ev_function_call_with_namespace;use core_test_support::responses::ev_response_created;@@ -1529,6 +1530,230 @@ async fn apps_guidance_and_deferred_namespace_appear_after_recovery_within_a_turOk(())}+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]+async fn code_mode_discovered_apps_tool_survives_startup_failure_and_recovers() -> Result<()> {+ skip_if_no_network!(Ok(()));++ let server = responses::start_mock_server().await;+ // Holding MCP initialization must not block inference or connector directory requests.+ let mcp_server = responses::start_mock_server().await;+ let (gated_apps, startup) = AppsTestServer::mount_with_startup_control(&mcp_server).await?;+ let apps = AppsTestServer::mount_searchable(&server).await?;+ let mut extensions = ExtensionRegistryBuilder::new();+ extensions.mcp_server_contributor(Arc::new(AppsMcpServerContributor {+ id: "hosted_plugin_runtime",+ url: format!("{}/api/codex/ps/mcp", gated_apps.chatgpt_base_url),+ root_resolved: None,+ }));+ let extensions = Arc::new(extensions.build());+ let code_mode_host = codex_utils_cargo_bin::cargo_bin("codex-code-mode-host")?;+ let builder = || {+ search_capable_apps_builder(apps.chatgpt_base_url.clone())+ .with_extensions(Arc::clone(&extensions))+ .with_code_mode_host_program(code_mode_host.clone())+ .with_model_info_override("gpt-5.5", |model| {+ model.tool_mode = Some(codex_protocol::openai_models::ToolMode::CodeMode);+ })+ .with_config(|config| {+ config.permissions.approval_policy = Constrained::allow_any(AskForApproval::Never);+ config+ .features+ .enable(Feature::CodeMode)+ .expect("test config should allow Code Mode");+ config+ .features+ .enable(Feature::CodeModeHost)+ .expect("test config should allow the Code Mode host");+ config+ .features+ .enable(Feature::DefaultModeRequestUserInput)+ .expect("test config should allow user input");+ })+ };+ let warm = builder().build_with_auto_env(&server).await?;+ wait_for_mcp_server(&warm.codex, CODEX_APPS_MCP_SERVER_NAME).await?;+ warm.codex.ensure_rollout_materialized().await;++ // Restart with the persisted catalog and hold discovery until the model has used it.+ // Queue recovery too: startup schedules its retry immediately after reporting failure.+ let initial_startup = startup.hold_next_successful_initialize();+ let recovery = startup.hold_next_successful_initialize();+ let test = builder().restart_with_auto_env(&server, &warm).await?;+ let question = json!({+ "questions": [{+ "id": "continue",+ "header": "Continue",+ "question": "Continue the tool call?",+ "options": [+ {"label": "Yes (Recommended)", "description": "Continue."},+ {"label": "No", "description": "Stop."}+ ]+ }]+ });+ let response = mount_sse_sequence(+ &server,+ vec![+ sse(vec![+ ev_response_created("discover"),+ ev_custom_tool_call(+ "discover",+ "exec",+ "const tool = ALL_TOOLS.find(t => t.name.endsWith('list_events'));\n\+ store('calendar-tool', tool.name); text(tool.name);",+ ),+ ev_completed("discover"),+ ]),+ sse(vec![+ ev_response_created("after-discovery"),+ ev_function_call(+ "after-discovery",+ "request_user_input",+ &question.to_string(),+ ),+ ev_completed("after-discovery"),+ ]),+ sse(vec;\n\+ yield_control();\n\+ text(await pending);",+ ),+ ev_completed("invoke"),+ ]),+ sse(vec![+ ev_response_created("after-invocation"),+ ev_function_call(+ "after-invocation",+ "request_user_input",+ &question.to_string(),+ ),+ ev_completed("after-invocation"),+ ]),+ ],+ )+ .await;+ test.codex+ .start_or_steer_turn(TurnInputRequest::user_input(vec![UserInput::Text {+ text: "Discover Calendar, then list events.".into(),+ text_elements: Vec::new(),+ }]))+ .await?;+ let EventMsg::RequestUserInput(discovered) = wait_for_event(&test.codex, |event| {+ assert!(+ !matches!(event, EventMsg::TurnComplete(_)),+ "turn ended before discovering the cached tool"+ );+ matches!(event, EventMsg::RequestUserInput(_))+ })+ .await+ else {+ unreachable!()+ };++ // Dropping the held response fails it; fail the transport retry as well.+ startup.fail_next_initialize_attempts(/*attempts*/ 1);+ drop(initial_startup);+ wait_for_event(&test.codex, |event| {+ matches!(+ event, EventMsg::McpStartupUpdate(update)+ if update.server == CODEX_APPS_MCP_SERVER_NAME+ && matches!(update.status, codex_protocol::protocol::McpStartupStatus::Failed { .. })+ )+ })+ .await;+ test.codex+ .submit(Op::UserInputAnswer {+ id: discovered.turn_id,+ response: RequestUserInputResponse {+ answers: HashMap::from([(+ "continue".to_string(),+ RequestUserInputAnswer {+ answers: vec!["Yes (Recommended)".to_string()],+ },+ )]),+ },+ })+ .await?;++ // This prompt is after the invocation in the second cell: the function must still exist,+ // and waiting for its connection must leave unrelated tools able to execute.+ let EventMsg::RequestUserInput(invoked) = wait_for_event(&test.codex, |event| {+ assert!(+ !matches!(event, EventMsg::TurnComplete(_)),+ "turn ended before invoking the cached tool"+ );+ matches!(event, EventMsg::RequestUserInput(_))+ })+ .await+ else {+ unreachable!()+ };+ let requests = response.requests();+ assert_eq!(requests.len(), 4);+ assert!(+ super::code_mode::custom_tool_output_last_non_empty_text(&requests[1], "discover")+ .expect("discovery should return the cached tool name")+ .ends_with("list_events")+ );+ let running = super::code_mode::custom_tool_output_last_non_empty_text(&requests[3], "invoke")+ .expect("the invocation should wait for Apps recovery");+ let cell_id = running+ .strip_prefix("Script running with cell ID ")+ .and_then(|text| text.lines().next())+ .expect("invoking the discovered function should yield, not fail");+ let finished = responses::mount_function_call_agent_response(+ &server,+ "finish",+ &json!({"cell_id": cell_id, "yield_time_ms": 10_000}).to_string(),+ "wait",+ )+ .await;+ recovery+ .send(())+ .expect("the shared Apps retry should still be waiting");+ test.codex+ .submit(Op::UserInputAnswer {+ id: invoked.turn_id,+ response: RequestUserInputResponse {+ answers: HashMap::from([(+ "continue".to_string(),+ RequestUserInputAnswer {+ answers: vec!["Yes (Recommended)".to_string()],+ },+ )]),+ },+ })+ .await?;+ wait_for_event(&test.codex, |event| {+ matches!(event, EventMsg::TurnComplete(_))+ })+ .await;++ let output = finished+ .completion+ .single_request()+ .function_call_output("finish");+ assert!(+ output["output"]+ .as_array()+ .expect("wait should return Code Mode content items")+ .iter()+ .any(|item| item["text"]+ .as_str()+ .is_some_and(|text| text.contains("called calendar_list_events"))),+ "{output}"+ );+ let calls = core_test_support::apps_test_server::recorded_apps_tool_calls(&mcp_server).await;+ assert_eq!(calls.len(), 1);+ assert_eq!(calls[0]["params"]["name"], "calendar_list_events");+ assert_eq!(startup.initialize_attempts(), 4);+ test.codex.shutdown_and_wait().await?;+ Ok(())+}+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]async fn later_follow_up_uses_background_recovered_apps_after_mid_thread_startup_failures()-> Result<()> {