Lightweight coding agent that runs in your terminal
Rust
Time out stalled code-mode host requests (#36830)
Adam Perry @ OpenAI8e3b5d3e875f
diff --git a/codex-rs/code-mode/src/remote_session/connection.rs b/codex-rs/code-mode/src/remote_session/connection.rsindex 30310f3e83..d08c8fbbc4 100644--- a/codex-rs/code-mode/src/remote_session/connection.rs+++ b/codex-rs/code-mode/src/remote_session/connection.rs@@ -1,4 +1,5 @@use std::fmt;+use std::future::Future;use std::io;use std::path::Path;use std::path::PathBuf;@@ -62,6 +63,8 @@ mod transport;const IPC_CHANNEL_CAPACITY: usize = 128;const HOST_HANDSHAKE_TIMEOUT: Duration = Duration::from_secs(10);+// TODO(anp) make this timeout configurable if 60 seconds is insufficient.+const DEFAULT_HOST_WAIT_TRANSPORT_TIMEOUT: Duration = Duration::from_secs(60);const MAX_WEBSOCKET_FRAME_BYTES: usize = MAX_FRAME_BYTES + std::mem::size_of::<u32>();// Host spawn errors become model-visible tool output. Bound configured paths// while preserving the executable-bearing suffix needed to diagnose failures.@@ -526,16 +529,23 @@ impl Connection {session: RemoteSession,request: WaitRequest,) -> Result<WaitOutcome, String> {+ // Account for the runtime's one-second yield grace separately from transport.+ let runtime_timeout =+ Duration::from_millis(request.yield_time_ms).saturating_add(Duration::from_secs(1));let cancellation = CallerCancellation::new();let (response_tx, response_rx) = oneshot::channel();- self.send(DriverCommand::Wait {- session,- request,- caller_cancellation: cancellation.token(),- response_tx,- })- .await?;- let result = self.receive(response_rx).await;+ let result = self+ .with_transport_deadline(runtime_timeout, "wait", async {+ self.send(DriverCommand::Wait {+ session,+ request,+ caller_cancellation: cancellation.token(),+ response_tx,+ })+ .await?;+ self.receive(response_rx).await+ })+ .await;cancellation.disarm();result}@@ -546,13 +556,16 @@ impl Connection {cell_id: CellId,) -> Result<WaitOutcome, String> {let (response_tx, response_rx) = oneshot::channel();- self.send(DriverCommand::Terminate {- session,- cell_id,- response_tx,+ self.with_transport_deadline(Duration::ZERO, "terminate", async {+ self.send(DriverCommand::Terminate {+ session,+ cell_id,+ response_tx,+ })+ .await?;+ self.receive(response_rx).await})- .await?;- self.receive(response_rx).await+ .await}pub(super) async fn shutdown_session(&self, session: RemoteSession) -> Result<(), String> {@@ -565,6 +578,26 @@ impl Connection {self.receive(response_rx).await}+ async fn with_transport_deadline<T>(+ &self,+ runtime_timeout: Duration,+ request_type: &str,+ request: impl Future<Output = Result<T, String>>,+ ) -> Result<T, String> {+ let deadline = runtime_timeout.saturating_add(DEFAULT_HOST_WAIT_TRANSPORT_TIMEOUT);+ match tokio::time::timeout(deadline, request).await {+ Ok(result) => result,+ Err(_) => {+ warn!(request_type, "code-mode host request exceeded its deadline");+ let reason =+ format!("code-mode host timed out waiting for {request_type} response");+ mark_connection_dead(&self.alive, &self.failure, reason.clone());+ self.cancellation.cancel();+ Err(reason)+ }+ }+ }+async fn send(&self, command: DriverCommand) -> Result<(), String> {if !self.is_alive() {return Err(self.failure_message());diff --git a/codex-rs/code-mode/src/remote_session/connection/driver_tests.rs b/codex-rs/code-mode/src/remote_session/connection/driver_tests.rsindex bb33002ec2..2dc84eeeec 100644--- a/codex-rs/code-mode/src/remote_session/connection/driver_tests.rs+++ b/codex-rs/code-mode/src/remote_session/connection/driver_tests.rs@@ -32,6 +32,8 @@ use tokio::sync::mpsc;use tokio::sync::oneshot;use tokio_util::sync::CancellationToken;+use super::super::Connection;+use super::super::DEFAULT_HOST_WAIT_TRANSPORT_TIMEOUT;use super::ConnectionDriver;use super::DriverCommand;use super::DriverEvent;@@ -46,6 +48,7 @@ struct DriverHarness {outgoing_rx: mpsc::Receiver<codex_code_mode_protocol::host::EncodedFrame>,cancellation: CancellationToken,alive: Arc<AtomicBool>,+ failure: Arc<StdMutex<Option<String>>>,driver_task: tokio::task::JoinHandle<()>,}@@ -56,6 +59,7 @@ impl DriverHarness {let (outgoing_tx, outgoing_rx) = mpsc::channel(/*max_capacity*/ 16);let cancellation = CancellationToken::new();let alive = Arc::new(AtomicBool::new(true));+ let failure = Arc::new(StdMutex::new(None));let (driver, execute_claim_tx) = ConnectionDriver::new(command_rx,event_rx,@@ -63,7 +67,7 @@ impl DriverHarness {outgoing_tx,DriverLifecycle {alive: Arc::clone(&alive),- failure: Arc::new(StdMutex::new(None)),+ failure: Arc::clone(&failure),cancellation: cancellation.clone(),},);@@ -75,6 +79,7 @@ impl DriverHarness {outgoing_rx,cancellation,alive,+ failure,driver_task,}}@@ -647,6 +652,7 @@ async fn concurrent_large_delegate_results_do_not_disconnect_a_backpressured_bullet (bulk_tx, mut bulk_rx) = mpsc::channel(MAX_PENDING_DELEGATE_CALLS);let cancellation = CancellationToken::new();let alive = Arc::new(AtomicBool::new(true));+ let failure = Arc::new(StdMutex::new(None));let (driver, execute_claim_tx) = ConnectionDriver::new(command_rx,event_rx,@@ -654,7 +660,7 @@ async fn concurrent_large_delegate_results_do_not_disconnect_a_backpressured_buloutgoing_tx,DriverLifecycle {alive: Arc::clone(&alive),- failure: Arc::new(StdMutex::new(None)),+ failure: Arc::clone(&failure),cancellation: cancellation.clone(),},);@@ -666,6 +672,7 @@ async fn concurrent_large_delegate_results_do_not_disconnect_a_backpressured_buloutgoing_rx,cancellation,alive,+ failure,driver_task,};let session = remote_session();@@ -1328,6 +1335,90 @@ async fn remote_wait_accepts_durations_longer_than_five_minutes() {);}+/// A stalled remote wait must expire even when its request never reaches the host.+#[tokio::test(start_paused = true)]+async fn queued_remote_wait_times_out_and_invalidates_the_connection() {+ let mut harness = DriverHarness::start();+ let session = remote_session();+ harness+ .open(session.clone(), Arc::new(RecordingDelegate::default()))+ .await;+ let _started = harness+ .start_cell(session.clone(), /*request_id*/ 2, "1")+ .await;+ let connection = Connection {+ command_tx: harness.command_tx.clone(),+ execute_claim_tx: harness.execute_claim_tx.clone(),+ alive: Arc::clone(&harness.alive),+ failure: Arc::clone(&harness.failure),+ cancellation: harness.cancellation.clone(),+ };+ let response = tokio::spawn(async move {+ let result = connection+ .wait(+ session,+ WaitRequest {+ cell_id: CellId::new("1".to_string()),+ yield_time_ms: 1,+ },+ )+ .await;+ (connection, result)+ });+ while harness.outgoing_rx.is_empty() {+ tokio::task::yield_now().await;+ }+ tokio::time::advance(DEFAULT_HOST_WAIT_TRANSPORT_TIMEOUT + Duration::from_secs(2)).await;++ let (connection, result) = response.await.expect("wait task");+ assert_eq!(+ result,+ Err("code-mode host timed out waiting for wait response".to_string())+ );+ assert!(!harness.alive.load(Ordering::Acquire));+ assert!(harness.cancellation.is_cancelled());+ drop(connection);+}++/// A stalled termination must expire and invalidate the same connection as a stalled wait.+#[tokio::test(start_paused = true)]+async fn queued_remote_termination_times_out_and_invalidates_the_connection() {+ let mut harness = DriverHarness::start();+ let session = remote_session();+ harness+ .open(session.clone(), Arc::new(RecordingDelegate::default()))+ .await;+ let _started = harness+ .start_cell(session.clone(), /*request_id*/ 2, "1")+ .await;+ let connection = Connection {+ command_tx: harness.command_tx.clone(),+ execute_claim_tx: harness.execute_claim_tx.clone(),+ alive: Arc::clone(&harness.alive),+ failure: Arc::clone(&harness.failure),+ cancellation: harness.cancellation.clone(),+ };+ let response = tokio::spawn(async move {+ let result = connection+ .terminate(session, CellId::new("1".to_string()))+ .await;+ (connection, result)+ });+ while harness.outgoing_rx.is_empty() {+ tokio::task::yield_now().await;+ }+ tokio::time::advance(DEFAULT_HOST_WAIT_TRANSPORT_TIMEOUT + Duration::from_secs(1)).await;++ let (connection, result) = response.await.expect("termination task");+ assert_eq!(+ result,+ Err("code-mode host timed out waiting for terminate response".to_string())+ );+ assert!(!harness.alive.load(Ordering::Acquire));+ assert!(harness.cancellation.is_cancelled());+ drop(connection);+}+#[tokio::test]async fn cancelled_wait_is_retired_before_next_wait_is_sent() {let mut harness = DriverHarness::start();diff --git a/codex-rs/core/tests/suite/code_mode.rs b/codex-rs/core/tests/suite/code_mode.rsindex 5ed90d2b62..1056b2f664 100644--- a/codex-rs/core/tests/suite/code_mode.rs+++ b/codex-rs/core/tests/suite/code_mode.rs@@ -21,6 +21,7 @@ use codex_protocol::dynamic_tools::DynamicToolNamespaceTool;use codex_protocol::dynamic_tools::DynamicToolResponse;use codex_protocol::dynamic_tools::DynamicToolSpec;use codex_protocol::models::PermissionProfile;+use codex_protocol::models::ResponseItem;use codex_protocol::openai_models::ToolMode;use codex_protocol::protocol::AskForApproval;use codex_protocol::protocol::EventMsg;@@ -2053,6 +2054,140 @@ try {Ok(())}+/// A stalled host wait must return its timeout to the model and reconnect for the next exec.+#[tokio::test(flavor = "current_thread")]+async fn code_mode_wait_timeout_reconnects_on_next_exec() -> Result<()> {+ skip_if_no_network!(Ok(()));++ let server = responses::start_mock_server().await;+ let mut builder = test_codex()+ .with_model("test-gpt-5.1-codex")+ .with_config(|config| {+ config+ .features+ .enable(Feature::CodeMode)+ .expect("code mode should be enabled");+ });+ let test = builder.build_with_auto_env(&server).await?;++ let first_turn = responses::mount_sse_sequence(+ &server,+ vec![+ sse(vec![+ ev_response_created("resp-1"),+ ev_custom_tool_call(+ "call-1",+ "exec",+ "yield_control(); await new Promise(() => {});",+ ),+ ev_completed("resp-1"),+ ]),+ sse(vec![+ ev_assistant_message("msg-1", "waiting"),+ ev_completed("resp-2"),+ ]),+ ],+ )+ .await;++ test.submit_turn("start a stalled code-mode cell").await?;+ let first_request = first_turn+ .last_request()+ .expect("initial exec should be returned to the model");+ let first_items = custom_tool_output_items(&first_request, "call-1");+ let cell_id = extract_running_cell_id(text_item(&first_items, /*index*/ 0));++ let timeout_completion = responses::mount_function_call_agent_response(+ &server,+ "call-2",+ &serde_json::to_string(&serde_json::json!({+ "cell_id": cell_id,+ "yield_time_ms": 60_000,+ }))?,+ "wait",+ )+ .await+ .completion;++ test.codex+ .submit(Op::UserInput {+ items: vec![UserInput::Text {+ text: "wait for the stalled cell".to_string(),+ text_elements: Vec::new(),+ }],+ final_output_json_schema: None,+ responsesapi_client_metadata: None,+ additional_context: Default::default(),+ thread_settings: Default::default(),+ })+ .await?;+ wait_for_event_match(&test.codex, |event| match event {+ EventMsg::RawResponseItem(raw) => match &raw.item {+ ResponseItem::FunctionCall { call_id, .. } if call_id == "call-2" => Some(()),+ _ => None,+ },+ _ => None,+ })+ .await;++ tokio::time::pause();+ for _ in 0..130 {+ tokio::time::advance(Duration::from_secs(1)).await;+ if timeout_completion+ .function_call_output_text("call-2")+ .is_some()+ {+ break;+ }+ }+ tokio::time::resume();+ wait_for_event(&test.codex, |event| {+ matches!(event, EventMsg::TurnComplete(_))+ })+ .await;++ let timeout_output = timeout_completion+ .function_call_output_text("call-2")+ .expect("timed-out wait should be returned to the model");+ assert!(+ timeout_output.contains("code-mode host timed out waiting for wait response"),+ "unexpected wait output: {timeout_output}"+ );++ let reconnect_turn = responses::mount_sse_sequence(+ &server,+ vec![+ sse(vec![+ ev_response_created("resp-5"),+ ev_custom_tool_call(+ "call-3",+ "exec",+ r#"text("reconnected"); yield_control(); await new Promise(() => {});"#,+ ),+ ev_completed("resp-5"),+ ]),+ sse(vec![+ ev_assistant_message("msg-3", "reconnected"),+ ev_completed("resp-6"),+ ]),+ ],+ )+ .await;++ test.submit_turn("run a cell after the timeout").await?;+ let reconnect_request = reconnect_turn+ .last_request()+ .expect("replacement exec should be returned to the model");+ let reconnect_items = custom_tool_output_items(&reconnect_request, "call-3");+ assert_eq!(+ extract_running_cell_id(text_item(&reconnect_items, /*index*/ 0)),+ "g2:1"+ );+ assert_eq!(text_item(&reconnect_items, /*index*/ 1), "reconnected");++ Ok(())+}+#[cfg_attr(windows, ignore = "no exec_command on Windows")]#[tokio::test(flavor = "multi_thread", worker_threads = 2)]async fn code_mode_can_yield_and_resume_with_wait() -> Result<()> {