Lightweight coding agent that runs in your terminal
Rust
Recover gRPC code-mode sessions after missing-session errors (#52235)
surajs-oai137f25f6439c
diff --git a/codex-rs/code-mode-host/tests/grpc.rs b/codex-rs/code-mode-host/tests/grpc.rsindex c2af0e3e85..8f584a2203 100644--- a/codex-rs/code-mode-host/tests/grpc.rs+++ b/codex-rs/code-mode-host/tests/grpc.rs@@ -52,6 +52,8 @@ mod large_tool_delegate;mod network_policy_tests;#[path = "support/recording_delegate.rs"]mod recording_delegate;+#[path = "grpc/session_recovery_tests.rs"]+mod session_recovery_tests;use host::HostHarness;use large_tool_delegate::LargeToolResultDelegate;diff --git a/codex-rs/code-mode-host/tests/grpc/session_recovery_tests.rs b/codex-rs/code-mode-host/tests/grpc/session_recovery_tests.rsnew file mode 100644index 0000000000..fa75218808--- /dev/null+++ b/codex-rs/code-mode-host/tests/grpc/session_recovery_tests.rs@@ -0,0 +1,101 @@+//! Exercises missing-session recovery while the host's lease streams remain open.++use std::sync::Arc;+use std::sync::Mutex;+use std::sync::atomic::AtomicUsize;+use std::sync::atomic::Ordering;++use anyhow::Result;+use codex_code_mode::CodeModeSessionProvider;+use codex_code_mode::GrpcCodeModeSessionProvider;+use codex_code_mode::NoopCodeModeSessionDelegate;+use codex_code_mode_host::GrpcCodeModeHost;+use codex_code_mode_protocol::grpc::code_mode_host_server::CodeModeHostServer;+use pretty_assertions::assert_eq;+use tokio::net::TcpListener;+use tokio_stream::wrappers::TcpListenerStream;+use tokio_util::task::AbortOnDropHandle;+use tonic::Code;+use tonic::Status;+use tonic::transport::Server;++use super::execute;+use super::request;+use super::text_response;++#[tokio::test]+async fn only_missing_sessions_reopen_without_replaying_execution() -> Result<()> {+ for (code, expected_cell, expected_value) in [+ (Code::NotFound, "g2:1", "undefined"),+ (Code::InvalidArgument, "2", "1"),+ ] {+ let rejection = Arc::new(Mutex::new(/*t*/ None));+ let next_rejection = Arc::clone(&rejection);+ let request_count = Arc::new(AtomicUsize::new(/*v*/ 0));+ let count = Arc::clone(&request_count);+ let service = CodeModeHostServer::with_interceptor(+ GrpcCodeModeHost::new(),+ move |request: tonic::Request<()>| {+ count.fetch_add(/*val*/ 1, Ordering::SeqCst);+ match next_rejection.lock().unwrap().take() {+ Some(rejection) => Err(rejection),+ None => Ok(request),+ }+ },+ );+ let listener = TcpListener::bind("127.0.0.1:0").await?;+ let endpoint = format!("http://{}", listener.local_addr()?);+ let _server = AbortOnDropHandle::new(tokio::spawn(+ Server::builder()+ .add_service(service)+ .serve_with_incoming(TcpListenerStream::new(listener)),+ ));+ let session = GrpcCodeModeSessionProvider::new(endpoint)+ .create_session()+ .await+ .map_err(anyhow::Error::msg)?;+ execute(+ &session,+ request(r#"store("before", 1);"#),+ Arc::new(NoopCodeModeSessionDelegate),+ )+ .await?;++ // Reject the next Execute without closing the lease or subscription streams.+ let failure = Status::new(code, "injected RPC rejection");+ *rejection.lock().unwrap() = Some(failure.clone());+ let requests_before_rejection = request_count.load(Ordering::SeqCst);+ let failed = execute(+ &session,+ request(r#"store("before", "replayed");"#),+ Arc::new(NoopCodeModeSessionDelegate),+ )+ .await+ .unwrap_err();+ assert_eq!(+ failed.to_string(),+ format!("gRPC code-mode execution failed: {failure}")+ );+ assert_eq!(+ request_count.load(Ordering::SeqCst),+ requests_before_rejection + 1,+ "the failed call must return without opening a session or replaying Execute"+ );+ let actual = execute(+ &session,+ request(r#"text(String(load("before")))"#),+ Arc::new(NoopCodeModeSessionDelegate),+ )+ .await?;+ assert_eq!(+ actual,+ text_response(+ expected_cell,+ expected_value,+ actual.code_mode_host_duration()+ )+ );+ session.shutdown().await.map_err(anyhow::Error::msg)?;+ }+ Ok(())+}diff --git a/codex-rs/code-mode-protocol/src/grpc/codex.code_mode.v1.proto b/codex-rs/code-mode-protocol/src/grpc/codex.code_mode.v1.protoindex 84fb22a373..4e4134c21c 100644--- a/codex-rs/code-mode-protocol/src/grpc/codex.code_mode.v1.proto+++ b/codex-rs/code-mode-protocol/src/grpc/codex.code_mode.v1.proto@@ -5,6 +5,10 @@ package codex.code_mode.v1;// Hosts stateful JavaScript execution and delegates nested tool calls to the// session owner. Large tool inputs and outputs stay off the session event// stream so independent HTTP/2 streams can make progress concurrently.+//+// Execute (including its stream), Wait, and Terminate reserve NOT_FOUND for an+// unknown session_id. Clients may retire the session binding on this status.+// Missing cells use WaitResponse.missing_cell instead.service CodeModeHost {// Opens a session lease. The first event is always SessionOpened; dropping// this stream closes the session and terminates its active cells.@@ -19,6 +23,7 @@ service CodeModeHost {// Each result receives its own HTTP/2 stream, preventing a large response// from blocking unrelated tool completions or session control events.+ // NOT_FOUND may identify a missing invocation rather than a missing session.rpc CompleteToolCall(CompleteToolCallRequest)returns (CompleteToolCallResponse);rpc AcknowledgeNotification(AcknowledgeNotificationRequest)diff --git a/codex-rs/code-mode/src/grpc_session/callbacks.rs b/codex-rs/code-mode/src/grpc_session/callbacks.rsindex 21f74cec0c..94a1631bc1 100644--- a/codex-rs/code-mode/src/grpc_session/callbacks.rs+++ b/codex-rs/code-mode/src/grpc_session/callbacks.rs@@ -193,6 +193,8 @@ impl SessionInner {tokio::select! {biased;_ = cancellation.cancelled() => {}+ // NotFound can mean an invocation in a live session was canceled.+ // Check cancellation before treating a completion error as fatal.result = deadline::request(self,"tool invocation completion",@@ -266,3 +268,7 @@ impl SessionInner {Ok(())}}++#[cfg(test)]+#[path = "callbacks_tests.rs"]+mod tests;diff --git a/codex-rs/code-mode/src/grpc_session/callbacks_tests.rs b/codex-rs/code-mode/src/grpc_session/callbacks_tests.rsnew file mode 100644index 0000000000..6351d3ce50--- /dev/null+++ b/codex-rs/code-mode/src/grpc_session/callbacks_tests.rs@@ -0,0 +1,50 @@+//! Keeps late tool-completion errors scoped to the canceled invocation.++use std::collections::HashMap;+use std::sync::Arc;+use std::sync::Mutex;+use std::sync::atomic::AtomicBool;++use pretty_assertions::assert_eq;+use tokio_util::sync::CancellationToken;+use tokio_util::task::TaskTracker;+use tower::service_fn;+use tower::util::BoxCloneSyncService;++use super::super::GrpcClient;+use super::super::state::SessionState;+use super::super::transport::SharedTransport;+use super::SessionInner;++#[tokio::test]+async fn missing_invocation_completion_racing_cancellation_preserves_session() {+ let cancellation = CancellationToken::new();+ let completion_cancellation = cancellation.clone();+ let client = GrpcClient::new(BoxCloneSyncService::new(service_fn(move |_request| {+ // Cancel after the biased select has polled cancellation, before the RPC fails.+ completion_cancellation.cancel();+ async { Ok(tonic::Status::not_found("unknown code-mode tool invocation").into_http()) }+ })));+ let session = SessionInner {+ id: "session".to_string(),+ client: client.clone(),+ runtime: tokio::runtime::Handle::current(),+ state: Mutex::new(SessionState::default()),+ wait_slots: Mutex::new(HashMap::new()),+ shutdown_requested: AtomicBool::new(/*v*/ false),+ shutdown_result: Mutex::new(/*t*/ None),+ stopped: CancellationToken::new(),+ stream_tasks: TaskTracker::new(),+ _transport: Arc::new(SharedTransport::Connected(client)),+ };++ session+ .complete_tool_call(+ "invocation".to_string(),+ cancellation,+ Ok(serde_json::Value::Null),+ )+ .await;++ assert_eq!(session.require_open(), Ok(()));+}diff --git a/codex-rs/code-mode/src/grpc_session/deadline.rs b/codex-rs/code-mode/src/grpc_session/deadline.rsindex 266b1ed5e5..f172343c84 100644--- a/codex-rs/code-mode/src/grpc_session/deadline.rs+++ b/codex-rs/code-mode/src/grpc_session/deadline.rs@@ -42,6 +42,25 @@ pub(super) async fn request<T>(}}+/// Invalidates the session on RPCs where `NotFound` means the session is missing.+pub(super) async fn session_request<T>(+ session: &SessionInner,+ operation: &str,+ runtime_timeout: Duration,+ rpc: impl Future<Output = Result<T, tonic::Status>>,+) -> Result<T, String> {+ request(session, operation, runtime_timeout, async {+ let result = rpc.await;+ if let Err(error) = &result+ && error.code() == tonic::Code::NotFound+ {+ session.fail(failure(operation, error));+ }+ result+ })+ .await+}+pub(super) fn failure(operation: &str, error: impl Display) -> String {let mut message = format!("gRPC code-mode {operation} failed: {error}");if message.len() > MAX_ERROR_BYTES {diff --git a/codex-rs/code-mode/src/grpc_session/operations.rs b/codex-rs/code-mode/src/grpc_session/operations.rsindex 922a036a3b..bd266bedf8 100644--- a/codex-rs/code-mode/src/grpc_session/operations.rs+++ b/codex-rs/code-mode/src/grpc_session/operations.rs@@ -130,11 +130,15 @@ impl SessionInner {{request.metadata_mut().insert("traceparent", traceparent);}- let mut stream =- deadline::request(&self, "execution", Duration::ZERO, client.execute(request))- .await?- .into_inner();- let first = deadline::request(+ let mut stream = deadline::session_request(+ &self,+ "execution",+ Duration::ZERO,+ client.execute(request),+ )+ .await?+ .into_inner();+ let first = deadline::session_request(&self,"execution starting event",Duration::ZERO,@@ -217,7 +221,7 @@ impl SessionInner {let outcome = tokio::select! {biased;_ = response_tx.closed() => return,- outcome = deadline::request(+ outcome = deadline::session_request(&self,"execution outcome",runtime_timeout,@@ -317,7 +321,8 @@ impl SessionInner {grpc::yield_observation_request::Observation::WaitId(wait_id),)});- let response = deadline::request(self, "wait", runtime_timeout, client.wait(request)).await;+ let response =+ deadline::session_request(self, "wait", runtime_timeout, client.wait(request)).await;drop(watcher);cancellation.disarm();self.prune_wait_slots();@@ -351,7 +356,7 @@ impl SessionInner {pub(super) async fn terminate(&self, cell_id: CellId) -> Result<WaitOutcome, String> {self.require_open()?;let mut client = self.client();- let response = deadline::request(+ let response = deadline::session_request(self,"termination",Duration::ZERO,