Skip to content

armgit / codex

Public Imported from GitHub
0

Lightweight coding agent that runs in your terminal

Rust

Finish idle thread unloads after slow shutdown (#51420)

jif946bb6c58398
diff --git a/codex-rs/app-server/src/request_processors/thread_lifecycle.rs b/codex-rs/app-server/src/request_processors/thread_lifecycle.rs
index b2b300ae40..34fff277d4 100644
--- a/codex-rs/app-server/src/request_processors/thread_lifecycle.rs
+++ b/codex-rs/app-server/src/request_processors/thread_lifecycle.rs
@@ -443,8 +443,24 @@ pub(super) async fn unload_thread_without_subscribers(
thread_state_manager.remove_thread_state(thread_id).await;
tokio::spawn(async move {
- match wait_for_thread_shutdown(&thread).await {
- ThreadShutdownResult::Complete => {
+ // The deadline bounds our warning, not background cleanup. Keep polling the
+ // same future so even delayed shutdown submission can eventually finish.
+ let shutdown = thread.shutdown_and_wait();
+ tokio::pin!(shutdown);
+ let result = match tokio::time::timeout(Duration::from_secs(/*secs*/ 10), &mut shutdown)
+ .await
+ {
+ Ok(result) => result,
+ Err(_) => {
+ warn!(
+ event.name = "codex.app_server.thread_shutdown_slow",
+ "thread {thread_id} shutdown is taking longer than expected; continuing to wait"
+ );
+ shutdown.await
+ }
+ };
+ match result {
+ Ok(()) => {
// A delayed unload can finish after thread/revert replaces this runtime under
// the same thread ID. Only the runtime that scheduled this unload may remove it.
if thread_manager
@@ -467,14 +483,10 @@ pub(super) async fn unload_thread_without_subscribers(
.await;
pending_thread_unloads.lock().await.remove(&thread_id);
}
- ThreadShutdownResult::SubmitFailed => {
+ Err(_) => {
pending_thread_unloads.lock().await.remove(&thread_id);
warn!("failed to submit Shutdown to thread {thread_id}");
}
- ThreadShutdownResult::TimedOut => {
- pending_thread_unloads.lock().await.remove(&thread_id);
- warn!("thread {thread_id} shutdown timed out; leaving thread loaded");
- }
}
});
}
diff --git a/codex-rs/app-server/tests/common/json_logging.rs b/codex-rs/app-server/tests/common/json_logging.rs
index 8683d1bdaf..6a854e6688 100644
--- a/codex-rs/app-server/tests/common/json_logging.rs
+++ b/codex-rs/app-server/tests/common/json_logging.rs
@@ -26,8 +26,14 @@ impl JsonLogCapture {
self.updated.notify_one();
}
- pub(crate) async fn wait_for_event(&self, event_name: &str) -> Result<Value> {
- let mut events = self.wait_for_events(event_name, /*count*/ 1).await?;
+ pub(crate) async fn wait_for_event(
+ &self,
+ event_name: &str,
+ timeout: Duration,
+ ) -> Result<Value> {
+ let mut events = self
+ .wait_for_events(event_name, /*count*/ 1, timeout)
+ .await?;
Ok(events.remove(0))
}
@@ -35,8 +41,9 @@ impl JsonLogCapture {
&self,
event_name: &str,
count: usize,
+ timeout: Duration,
) -> Result<Vec<Value>> {
- let result = tokio::time::timeout(Duration::from_secs(10), async {
+ let result = tokio::time::timeout(timeout, async {
loop {
let updated = self.updated.notified();
let events = self
diff --git a/codex-rs/app-server/tests/common/test_app_server.rs b/codex-rs/app-server/tests/common/test_app_server.rs
index 86044c7c71..51276510f2 100644
--- a/codex-rs/app-server/tests/common/test_app_server.rs
+++ b/codex-rs/app-server/tests/common/test_app_server.rs
@@ -243,12 +243,13 @@ impl TestAppServer {
})
}
- /// Waits for a JSON stderr event whose structured `event.name` field matches.
+ /// Waits up to `timeout` for a JSON stderr event whose structured `event.name` field matches.
pub async fn wait_for_json_log_event(
&self,
event_name: &str,
+ timeout: Duration,
) -> anyhow::Result<serde_json::Value> {
- self.json_logs.wait_for_event(event_name).await
+ self.json_logs.wait_for_event(event_name, timeout).await
}
async fn new_with_program_env_and_args(
diff --git a/codex-rs/app-server/tests/suite/logging.rs b/codex-rs/app-server/tests/suite/logging.rs
index 67da665568..1d6d49c777 100644
--- a/codex-rs/app-server/tests/suite/logging.rs
+++ b/codex-rs/app-server/tests/suite/logging.rs
@@ -625,7 +625,7 @@ async fn app_server_emits_structured_tool_call_timing_event() -> Result<()> {
.await??;
let mut tool_call = app_server
- .wait_for_json_log_event("codex.tool_call")
+ .wait_for_json_log_event("codex.tool_call", Duration::from_secs(/*secs*/ 10))
.await?;
let tool_call_object = tool_call
.as_object_mut()
diff --git a/codex-rs/app-server/tests/suite/v2/code_mode_host.rs b/codex-rs/app-server/tests/suite/v2/code_mode_host.rs
index 7ee4c12d2f..116311a822 100644
--- a/codex-rs/app-server/tests/suite/v2/code_mode_host.rs
+++ b/codex-rs/app-server/tests/suite/v2/code_mode_host.rs
@@ -149,7 +149,10 @@ async fn code_mode_model_output_uses_structured_host_timing(
.await?;
let _: TurnStartResponse = app_server.read_response(start_id).await?;
let host_timing = app_server
- .wait_for_json_log_event("codex.code_mode.host_timing")
+ .wait_for_json_log_event(
+ "codex.code_mode.host_timing",
+ Duration::from_secs(/*secs*/ 10),
+ )
.await?;
// Hold only the app-server after the host outcome, so local elapsed time
// cannot round to the same displayed duration as the host measurement.
@@ -179,7 +182,7 @@ async fn code_mode_model_output_uses_structured_host_timing(
assert_eq!(host_fields["call_id"], "timed-call");
assert_eq!(host_fields["tool_name"], tool_name);
let handler_timing = app_server
- .wait_for_json_log_event("codex.tool_call")
+ .wait_for_json_log_event("codex.tool_call", Duration::from_secs(/*secs*/ 10))
.await?;
let handler_fields = &handler_timing["fields"];
let handler_duration_ms = handler_fields["handler_duration_ms"]
diff --git a/codex-rs/app-server/tests/suite/v2/mod.rs b/codex-rs/app-server/tests/suite/v2/mod.rs
index 4eed47fe52..75855aad4d 100644
--- a/codex-rs/app-server/tests/suite/v2/mod.rs
+++ b/codex-rs/app-server/tests/suite/v2/mod.rs
@@ -156,6 +156,7 @@ mod thread_start;
mod thread_status;
mod thread_timeline;
mod thread_unarchive;
+mod thread_unload;
mod thread_unsubscribe;
mod turn_cost_otel;
mod turn_interrupt;
diff --git a/codex-rs/app-server/tests/suite/v2/otel.rs b/codex-rs/app-server/tests/suite/v2/otel.rs
index 1f5836dde6..e6ba7144b4 100644
--- a/codex-rs/app-server/tests/suite/v2/otel.rs
+++ b/codex-rs/app-server/tests/suite/v2/otel.rs
@@ -101,7 +101,10 @@ async fn account_switch_reloads_telemetry_collectors_and_preserves_trace_context
assert_eq!(response, LoginAccountResponse::ChatgptAuthTokens {});
timeout(
TEST_TIMEOUT,
- app_server.wait_for_json_log_event("codex.app_server.otel_reloaded"),
+ app_server.wait_for_json_log_event(
+ "codex.app_server.otel_reloaded",
+ Duration::from_secs(/*secs*/ 10),
+ ),
)
.await??;
timeout(TEST_TIMEOUT, async {
diff --git a/codex-rs/app-server/tests/suite/v2/thread_unload.rs b/codex-rs/app-server/tests/suite/v2/thread_unload.rs
new file mode 100644
index 0000000000..8c74cbf2b9
--- /dev/null
+++ b/codex-rs/app-server/tests/suite/v2/thread_unload.rs
@@ -0,0 +1,107 @@
+//! Idle unloading must finish even when resource teardown exceeds the shutdown warning deadline.
+
+use anyhow::Context;
+use anyhow::Result;
+use app_test_support::MockResponsesConfig;
+use app_test_support::TestAppServer;
+use app_test_support::create_mock_responses_server_sequence_unchecked;
+use codex_app_server_protocol::ClientRequest;
+use codex_app_server_protocol::ThreadClosedNotification;
+use codex_app_server_protocol::ThreadLoadedListParams;
+use codex_app_server_protocol::ThreadLoadedListResponse;
+use codex_app_server_protocol::ThreadStartParams;
+use codex_app_server_protocol::ThreadUnsubscribeParams;
+use codex_app_server_protocol::ThreadUnsubscribeResponse;
+use codex_app_server_protocol::ThreadUnsubscribeStatus;
+use pretty_assertions::assert_eq;
+use std::fs::OpenOptions;
+use std::time::Duration;
+use tempfile::TempDir;
+use tokio::time::timeout;
+
+#[tokio::test]
+async fn idle_unload_finishes_after_slow_writer_shutdown() -> Result<()> {
+ let server = create_mock_responses_server_sequence_unchecked(Vec::new()).await;
+ let home = TempDir::new()?;
+ MockResponsesConfig::new(&server.uri())
+ .with_root_config("thread_unload_delay_secs = 0\napprovals_reviewer = \"user\"")
+ .write(home.path())?;
+ let mut client = TestAppServer::builder()
+ .with_codex_home(home.path())
+ .with_json_logging("warn")
+ // Writer cleanup blocks one worker while the test queries the server.
+ .with_env_overrides(&[("TOKIO_WORKER_THREADS", Some("2"))])
+ .build_initialized()
+ .await?;
+ let started = client.start_thread(ThreadStartParams::default()).await?;
+ let thread_id = started.thread.id;
+
+ // Block the real cross-process writer-lock cleanup. Other app-server tasks,
+ // including the ten-second shutdown deadline and JSON-RPC, can still run.
+ let coordination_lock = OpenOptions::new()
+ .read(true)
+ .write(true)
+ .open(home.path().join("thread-writer-locks/.coordination.lock"))?;
+ coordination_lock.lock()?;
+ let unsubscribed: ThreadUnsubscribeResponse = client
+ .request(|request_id| ClientRequest::ThreadUnsubscribe {
+ request_id,
+ params: ThreadUnsubscribeParams {
+ thread_id: thread_id.clone(),
+ },
+ })
+ .await?;
+ assert_eq!(
+ unsubscribed,
+ ThreadUnsubscribeResponse {
+ status: ThreadUnsubscribeStatus::Unsubscribed
+ }
+ );
+ // Unsubscribe only schedules shutdown. Keep the lock until the shutdown
+ // future has actually crossed its warning deadline.
+ client
+ .wait_for_json_log_event(
+ "codex.app_server.thread_shutdown_slow",
+ Duration::from_secs(/*secs*/ 30),
+ )
+ .await
+ .context("idle unload never reached the shutdown warning deadline")?;
+ let loaded: ThreadLoadedListResponse = client
+ .request(|request_id| ClientRequest::ThreadLoadedList {
+ request_id,
+ params: ThreadLoadedListParams::default(),
+ })
+ .await?;
+ assert_eq!(
+ loaded,
+ ThreadLoadedListResponse {
+ data: vec![thread_id.clone()],
+ next_cursor: None
+ }
+ );
+
+ // Once persistence can finish, delayed shutdown must still remove the runtime
+ // and report closure without another client subscription or unload request.
+ drop(coordination_lock);
+ let closed: ThreadClosedNotification = timeout(
+ Duration::from_secs(/*secs*/ 10),
+ client.read_notification("thread/closed"),
+ )
+ .await
+ .context("slow shutdown never finalized idle unload")??;
+ assert_eq!(closed, ThreadClosedNotification { thread_id });
+ let loaded: ThreadLoadedListResponse = client
+ .request(|request_id| ClientRequest::ThreadLoadedList {
+ request_id,
+ params: ThreadLoadedListParams::default(),
+ })
+ .await?;
+ assert_eq!(
+ loaded,
+ ThreadLoadedListResponse {
+ data: Vec::new(),
+ next_cursor: None
+ }
+ );
+ Ok(())
+}