Skip to content

armgit / codex

Public Imported from GitHub
0

Lightweight coding agent that runs in your terminal

Rust

Add live WebRTC voice conversations to the TUI (#43581)

Benjamin Carlssonb01c3986fd2e
diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock
index 3af6001c9d..6f422e751f 100644
--- a/codex-rs/Cargo.lock
+++ b/codex-rs/Cargo.lock
@@ -4677,6 +4677,7 @@ dependencies = [
"codex-otel",
"codex-plugin",
"codex-protocol",
+ "codex-realtime-webrtc",
"codex-rollout",
"codex-sandboxing",
"codex-shell-command",
diff --git a/codex-rs/core/src/realtime_conversation.rs b/codex-rs/core/src/realtime_conversation.rs
index a810a48c17..63f59d5408 100644
--- a/codex-rs/core/src/realtime_conversation.rs
+++ b/codex-rs/core/src/realtime_conversation.rs
@@ -76,6 +76,7 @@ use std::sync::atomic::Ordering;
use std::time::Duration;
use std::time::Instant;
use tokio::sync::Mutex;
+use tokio::sync::Semaphore;
use tokio::task::JoinHandle;
use tokio_util::sync::CancellationToken;
use tracing::debug;
@@ -129,6 +130,37 @@ enum RealtimeFanoutTaskStop {
Detach,
}
+/// Serializes turn admission with the safety cutoff for one voice session.
+struct RealtimeHandoffAdmission {
+ gate: Semaphore,
+ retired: AtomicBool,
+}
+
+impl RealtimeHandoffAdmission {
+ fn new() -> Self {
+ Self {
+ gate: Semaphore::new(1),
+ retired: AtomicBool::new(false),
+ }
+ }
+
+ async fn route(&self, session: &Arc<Session>, text: String) {
+ let Ok(_permit) = self.gate.acquire().await else {
+ return;
+ };
+ if !self.retired.load(Ordering::Acquire) {
+ session.route_realtime_text_input(text).await;
+ }
+ }
+
+ async fn retire(&self) {
+ let Ok(_permit) = self.gate.acquire().await else {
+ return;
+ };
+ self.retired.store(true, Ordering::Release);
+ }
+}
+
pub(crate) struct RealtimeConversationManager {
state: Mutex<Option<ConversationState>>,
mode_instructions: Mutex<Option<RealtimeModeInstructions>>,
@@ -467,6 +499,8 @@ struct ConversationState {
input_task: JoinHandle<()>,
fanout_task: Option<JoinHandle<()>>,
realtime_active: Arc<AtomicBool>,
+ // A misalignment failure retires only this session's queued handoffs.
+ route_handoffs: Arc<RealtimeHandoffAdmission>,
stop_token: CancellationToken,
}
@@ -489,6 +523,7 @@ struct RealtimeStart {
struct RealtimeStartOutput {
realtime_active: Arc<AtomicBool>,
+ route_handoffs: Arc<RealtimeHandoffAdmission>,
events_rx: Receiver<RealtimeEvent>,
transcript_tail_rx: Receiver<String>,
sdp: Option<String>,
@@ -529,6 +564,20 @@ impl RealtimeConversationManager {
)
}
+ /// Prevent already parsed handoffs from starting turns after a safety failure.
+ /// Keep the gate on the old session so a later voice session starts fresh.
+ pub(crate) async fn retire_handoffs_for_misalignment(&self) {
+ let route_handoffs = self
+ .state
+ .lock()
+ .await
+ .as_ref()
+ .map(|state| Arc::clone(&state.route_handoffs));
+ if let Some(route_handoffs) = route_handoffs {
+ route_handoffs.retire().await;
+ }
+ }
+
async fn start(
&self,
start: RealtimeStart,
@@ -581,6 +630,7 @@ impl RealtimeConversationManager {
let (transcript_tail_tx, transcript_tail_rx) = async_channel::bounded::<String>(1);
let realtime_active = Arc::new(AtomicBool::new(true));
+ let route_handoffs = Arc::new(RealtimeHandoffAdmission::new());
let stop_token = CancellationToken::new();
let handoff = RealtimeHandoffState {
output_tx: handoff_output_tx,
@@ -690,10 +740,12 @@ impl RealtimeConversationManager {
input_task: task,
fanout_task: None,
realtime_active: Arc::clone(&realtime_active),
+ route_handoffs: Arc::clone(&route_handoffs),
stop_token,
});
Ok(RealtimeStartOutput {
realtime_active,
+ route_handoffs,
events_rx,
transcript_tail_rx,
sdp,
@@ -1571,6 +1623,7 @@ async fn handle_start_inner(
let RealtimeStartOutput {
realtime_active,
+ route_handoffs,
events_rx,
transcript_tail_rx,
sdp,
@@ -1597,10 +1650,9 @@ async fn handle_start_inner(
match &event {
RealtimeEvent::AudioOut(_) => {}
_ => {
- info!(
- event = ?event,
- "received realtime conversation event"
- );
+ // Transcripts, handoffs, and conversation items may contain
+ // credentials. Keep this receipt free of event payloads.
+ info!("received realtime conversation event");
}
}
if let RealtimeEvent::Error(_) = &event {
@@ -1613,9 +1665,9 @@ async fn handle_start_inner(
_ => None,
};
if let Some(text) = maybe_routed_text {
- debug!(text = %text, "[realtime-text] realtime conversation text output");
- let sess_for_routed_text = Arc::clone(&sess_clone);
- sess_for_routed_text.route_realtime_text_input(text).await;
+ // The routed text can contain spoken prompts or workspace secrets.
+ debug!("[realtime-text] realtime conversation text output");
+ route_handoffs.route(&sess_clone, text).await;
}
sess_clone
.send_event_raw(ev(EventMsg::RealtimeConversationRealtime(
@@ -1626,7 +1678,7 @@ async fn handle_start_inner(
.await;
}
if let Ok(text) = transcript_tail_rx.recv().await {
- sess_clone.route_realtime_text_input(text).await;
+ route_handoffs.route(&sess_clone, text).await;
}
if fanout_realtime_active.swap(false, Ordering::Relaxed) {
match end {
@@ -1785,7 +1837,7 @@ pub(crate) async fn handle_speech(
sub_id: String,
params: ConversationSpeechParams,
) {
- debug!(text = %params.text, "[realtime-text] appending realtime speech");
+ debug!("[realtime-text] appending realtime speech");
if let Err(err) = sess.conversation.append_speech(params.text).await {
error!("failed to append realtime speech: {err}");
if sess.conversation.running_state().await.is_some() {
diff --git a/codex-rs/core/src/session/turn.rs b/codex-rs/core/src/session/turn.rs
index 4719e1d891..7f31667988 100644
--- a/codex-rs/core/src/session/turn.rs
+++ b/codex-rs/core/src/session/turn.rs
@@ -636,6 +636,12 @@ pub(crate) async fn run_turn(
}
Err(e) => {
info!("Turn error: {e:#}");
+ if matches!(
+ e.details(),
+ CodexErrorDetails::MisalignmentPolicyViolation { .. }
+ ) {
+ sess.conversation.retire_handoffs_for_misalignment().await;
+ }
let error = e.to_codex_protocol_error();
sess.emit_turn_error_lifecycle(turn_context.as_ref(), error.clone())
.await;
diff --git a/codex-rs/core/tests/suite/mod.rs b/codex-rs/core/tests/suite/mod.rs
index 1c6a629926..1c23e2aba9 100644
--- a/codex-rs/core/tests/suite/mod.rs
+++ b/codex-rs/core/tests/suite/mod.rs
@@ -128,6 +128,7 @@ mod prompt_debug_tests;
mod quota_exceeded;
mod realtime_conversation;
mod realtime_initial_items;
+mod realtime_misalignment;
mod realtime_sideband_endpoint;
mod reasoning_effort_override;
mod remote_env;
diff --git a/codex-rs/core/tests/suite/realtime_misalignment.rs b/codex-rs/core/tests/suite/realtime_misalignment.rs
new file mode 100644
index 0000000000..109e61b78c
--- /dev/null
+++ b/codex-rs/core/tests/suite/realtime_misalignment.rs
@@ -0,0 +1,151 @@
+//! A failed turn must not admit handoffs still arriving from its voice session.
+
+use anyhow::Context;
+use anyhow::Result;
+use codex_config::config_toml::RealtimeWsVersion;
+use codex_protocol::protocol::CodexErrorInfo;
+use codex_protocol::protocol::ConversationStartParams;
+use codex_protocol::protocol::EventMsg;
+use codex_protocol::protocol::Op;
+use codex_protocol::protocol::RealtimeConversationRealtimeEvent;
+use codex_protocol::protocol::RealtimeEvent;
+use codex_protocol::protocol::RealtimeOutputModality;
+use core_test_support::responses;
+use core_test_support::skip_if_no_network;
+use core_test_support::test_codex::test_codex;
+use core_test_support::wait_for_event_match;
+use futures::SinkExt;
+use pretty_assertions::assert_eq;
+use serde_json::json;
+use std::time::Duration;
+use tokio::net::TcpListener;
+use tokio::sync::oneshot;
+use tokio_tungstenite::accept_async;
+use tokio_tungstenite::tungstenite::Message;
+use wiremock::ResponseTemplate;
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn misalignment_retires_late_voice_handoff_before_it_starts_a_turn() -> Result<()> {
+ skip_if_no_network!(Ok(()));
+
+ let api_server = responses::start_mock_server().await;
+ let first_response = responses::mount_response_once(
+ &api_server,
+ ResponseTemplate::new(403).set_body_json(json!({
+ "error": {
+ "message": "This request violated the misalignment policy.",
+ "type": "invalid_request_error",
+ "code": "misalignment_policy_violation"
+ }
+ })),
+ )
+ .await;
+
+ let listener = TcpListener::bind("127.0.0.1:0").await?;
+ let realtime_url = format!("ws://{}", listener.local_addr()?);
+ let (late_handoff_tx, mut late_handoff_rx) = oneshot::channel();
+ let (finish_tx, finish_rx) = oneshot::channel();
+ let sideband = tokio::spawn(async move {
+ let (stream, _) = listener.accept().await?;
+ let mut websocket = accept_async(stream).await?;
+ websocket
+ .send(Message::Text(
+ json!({
+ "type": "session.updated",
+ "session": {"id": "session_misalignment", "instructions": "backend prompt"}
+ })
+ .to_string()
+ .into(),
+ ))
+ .await?;
+ for (handoff_id, text) in [
+ ("initial_handoff", "first request"),
+ ("late_handoff", "must not start a turn"),
+ ] {
+ if handoff_id == "late_handoff" {
+ (&mut late_handoff_rx).await?;
+ }
+ websocket
+ .send(Message::Text(
+ json!({
+ "type": "conversation.handoff.requested",
+ "handoff_id": handoff_id,
+ "item_id": handoff_id,
+ "input_transcript": text
+ })
+ .to_string()
+ .into(),
+ ))
+ .await?;
+ }
+ finish_rx.await?;
+ websocket.close(None).await?;
+ Ok::<_, anyhow::Error>(())
+ });
+
+ let mut builder = test_codex().with_config(move |config| {
+ config.experimental_realtime_ws_base_url = Some(realtime_url);
+ config.realtime.version = RealtimeWsVersion::V1;
+ });
+ let test = builder.build_with_auto_env(&api_server).await?;
+ test.codex
+ .submit(Op::RealtimeConversationStart(ConversationStartParams {
+ client_managed_handoffs: false,
+ delegation_ack_filler: None,
+ flush_transcript_tail_on_session_end: false,
+ codex_responses_as_items: false,
+ codex_response_item_prefix: None,
+ codex_response_handoff_mode:
+ codex_protocol::protocol::CodexResponseHandoffMode::Thinking,
+ codex_response_handoff_channel_prefixes: None,
+ model: None,
+ output_modality: RealtimeOutputModality::Audio,
+ include_startup_context: true,
+ initial_items: Vec::new(),
+ realtime_start_instructions: None,
+ realtime_end_instructions: None,
+ prompt: Some(Some("backend prompt".to_string())),
+ realtime_session_id: None,
+ transport: None,
+ version: None,
+ voice: None,
+ }))
+ .await?;
+
+ wait_for_event_match(&test.codex, |event| match event {
+ EventMsg::Error(error)
+ if error.codex_error_info == Some(CodexErrorInfo::MisalignmentPolicyViolation) =>
+ {
+ Some(())
+ }
+ _ => None,
+ })
+ .await;
+ assert_eq!(first_response.requests().len(), 1);
+ late_handoff_tx.send(()).expect("sideband still open");
+ wait_for_event_match(&test.codex, |event| match event {
+ EventMsg::RealtimeConversationRealtime(RealtimeConversationRealtimeEvent {
+ payload: RealtimeEvent::HandoffRequested(handoff),
+ }) if handoff.handoff_id == "late_handoff" => Some(()),
+ _ => None,
+ })
+ .await;
+ finish_tx.send(()).expect("sideband still open");
+ sideband.await??;
+
+ // The fanout forwards a handoff only after deciding whether to route it.
+ // Give any spawned turn a chance to reach the mock API before checking.
+ tokio::time::sleep(Duration::from_millis(200)).await;
+ let requests = api_server
+ .received_requests()
+ .await
+ .context("API requests")?;
+ assert_eq!(
+ requests
+ .iter()
+ .filter(|request| request.url.path() == "/v1/responses")
+ .count(),
+ 1,
+ );
+ Ok(())
+}
diff --git a/codex-rs/realtime-webrtc/src/client.rs b/codex-rs/realtime-webrtc/src/client.rs
index 371c51bdfc..5f5138ecff 100644
--- a/codex-rs/realtime-webrtc/src/client.rs
+++ b/codex-rs/realtime-webrtc/src/client.rs
@@ -19,6 +19,22 @@ use crate::message_reader::MessageReader;
const DEADLINE: Duration = Duration::from_secs(/*secs*/ 5);
const RUNTIME_INITIALIZATION_DEADLINE: Duration = Duration::from_secs(/*secs*/ 30);
+/// Startup failures eligible for recovery remain distinct from all other failures.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub enum ConnectionError {
+ NegotiationTimedOut,
+ Failed,
+}
+impl std::fmt::Display for ConnectionError {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ f.write_str(match self {
+ Self::NegotiationTimedOut => "voice negotiation timed out",
+ Self::Failed => "voice connection failed",
+ })
+ }
+}
+impl std::error::Error for ConnectionError {}
+
/// Owns one helper. Dropping it terminates the process and leaves its waiter to reap it.
/// A successful handshake establishes compatibility only, not an active audio session.
pub struct VoiceHost {
@@ -94,6 +110,12 @@ impl VoiceHost {
Duration::from_secs(/*secs*/ 20),
)
.await?;
+ if response == (Message::TransportTimedOut {}) {
+ self.process.terminate();
+ // A lost exit notification or failed cleanup is not retryable.
+ timeout(DEADLINE, &mut self.exit).await??;
+ return Err(ConnectionError::NegotiationTimedOut.into());
+ }
ensure!(
response == Message::TransportReady {},
"unexpected voice helper response"
diff --git a/codex-rs/realtime-webrtc/src/client_tests.rs b/codex-rs/realtime-webrtc/src/client_tests.rs
index b5256bee47..e0c96ab7f3 100644
--- a/codex-rs/realtime-webrtc/src/client_tests.rs
+++ b/codex-rs/realtime-webrtc/src/client_tests.rs
@@ -61,3 +61,52 @@ async fn cancelling_initialization_terminates_the_owned_helper() -> anyhow::Resu
tokio::time::timeout(std::time::Duration::from_secs(/*secs*/ 1), spawned.exit_rx).await??;
Ok(())
}
+
+#[cfg(unix)]
+#[tokio::test]
+async fn negotiation_timeout_waits_for_helper_exit_before_allowing_retry() -> anyhow::Result<()> {
+ let spawned = codex_utils_pty::spawn_pipe_process(
+ std::path::Path::new("/bin/sleep"),
+ &["30".to_owned()],
+ std::path::Path::new("/"),
+ &child_environment(std::iter::empty()),
+ /*arg0*/ &None,
+ &[],
+ )
+ .await?;
+ let (output, receiver) = tokio::sync::mpsc::channel(/*buffer*/ 1);
+ output
+ .send(crate::encode_frame(&crate::Message::TransportTimedOut {})?)
+ .await?;
+ let (reaped, exit) = tokio::sync::oneshot::channel();
+ let host = super::VoiceHost {
+ process: spawned.session,
+ output: crate::message_reader::MessageReader::new(receiver),
+ exit,
+ };
+ let answer = crate::SessionDescription::try_from("synthetic-answer".to_owned()).unwrap();
+ let mut connection = Box::pin(host.apply_answer(answer));
+ std::future::poll_fn(|cx| {
+ assert!(std::future::Future::poll(connection.as_mut(), cx).is_pending());
+ std::task::Poll::Ready(())
+ })
+ .await;
+ let code =
+ tokio::time::timeout(std::time::Duration::from_secs(/*secs*/ 1), spawned.exit_rx).await??;
+ // The process has exited, but the explicit reaping acknowledgement is still required.
+ std::future::poll_fn(|cx| {
+ assert!(std::future::Future::poll(connection.as_mut(), cx).is_pending());
+ std::task::Poll::Ready(())
+ })
+ .await;
+ reaped.send(code).unwrap();
+ let error = connection
+ .await
+ .err()
+ .expect("timeout must not return a connected helper");
+ assert_eq!(
+ error.downcast_ref::<crate::ConnectionError>(),
+ Some(&crate::ConnectionError::NegotiationTimedOut)
+ );
+ Ok(())
+}
diff --git a/codex-rs/realtime-webrtc/src/lib.rs b/codex-rs/realtime-webrtc/src/lib.rs
index c65d724755..8abfa53eb3 100644
--- a/codex-rs/realtime-webrtc/src/lib.rs
+++ b/codex-rs/realtime-webrtc/src/lib.rs
@@ -3,6 +3,7 @@ mod message_reader;
mod protocol;
mod session;
+pub use client::ConnectionError;
pub use client::VoiceHost;
pub use protocol::AudioControls;
pub use protocol::AudioState;
diff --git a/codex-rs/realtime-webrtc/src/protocol.rs b/codex-rs/realtime-webrtc/src/protocol.rs
index f94bf53bae..2859aaea87 100644
--- a/codex-rs/realtime-webrtc/src/protocol.rs
+++ b/codex-rs/realtime-webrtc/src/protocol.rs
@@ -81,6 +81,7 @@ pub enum Message {
Offer { sdp: SessionDescription },
ApplyAnswer { sdp: SessionDescription },
TransportReady {},
+ TransportTimedOut {},
OpenDevices {},
DevicesOpened {},
SetAudioControls { controls: AudioControls },
diff --git a/codex-rs/realtime-webrtc/src/session.rs b/codex-rs/realtime-webrtc/src/session.rs
index 1241a40fa5..b699c03e4c 100644
--- a/codex-rs/realtime-webrtc/src/session.rs
+++ b/codex-rs/realtime-webrtc/src/session.rs
@@ -26,7 +26,10 @@ const ANSWER_WAIT: Duration = Duration::from_secs(/*secs*/ 40);
static RUNTIME: Mutex<Option<Arc<tokio::runtime::Runtime>>> = Mutex::new(None);
enum Command {
- Answer(SessionDescription, blocking::SyncSender<()>),
+ Answer(
+ SessionDescription,
+ blocking::SyncSender<std::result::Result<(), crate::ConnectionError>>,
+ ),
Controls(AudioControls),
}
@@ -196,15 +199,22 @@ impl RealtimeWebrtcSessionHandle {
}
/// Called off the UI thread; return only after negotiation, device startup and restored controls.
- pub fn apply_answer_sdp(&self, answer: String) -> Result<()> {
- let sdp = SessionDescription::try_from(answer).map_err(anyhow::Error::msg)?;
+ pub fn apply_answer_sdp(
+ &self,
+ answer: String,
+ ) -> std::result::Result<(), crate::ConnectionError> {
+ let sdp =
+ SessionDescription::try_from(answer).map_err(|_| crate::ConnectionError::Failed)?;
let (complete, result) = blocking::sync_channel(/*bound*/ 1);
- self.send(Command::Answer(sdp, complete))?;
- if result.recv_timeout(ANSWER_WAIT).is_err() {
- self.close();
- anyhow::bail!("voice connection failed");
+ self.send(Command::Answer(sdp, complete))
+ .map_err(|_| crate::ConnectionError::Failed)?;
+ match result.recv_timeout(ANSWER_WAIT) {
+ Ok(result) => result,
+ Err(_) => {
+ self.close();
+ Err(crate::ConnectionError::Failed)
+ }
}
- Ok(())
}
pub fn take_error(&self) -> Option<String> {
@@ -254,13 +264,23 @@ async fn run(
biased;
command = commands.recv() => match command {
Some(Command::Answer(sdp, complete)) if !connected => {
- host = host.apply_answer(sdp).await?.open_devices().await?;
+ host = match host.apply_answer(sdp).await {
+ Ok(host) => host,
+ Err(error) => {
+ let failure = error.downcast_ref::<crate::ConnectionError>()
+ .copied().unwrap_or(crate::ConnectionError::Failed);
+ let _ = complete.send(Err(failure));
+ // This failure is delivered by the startup completion only.
+ return Ok(());
+ }
+ };
+ host = host.open_devices().await?;
let applied = startup_controls(&mut commands, controls, |initial| {
host.begin_audio_controls(initial)
})?;
applied.await?;
connected = true;
- let _ = complete.send(());
+ let _ = complete.send(Ok(()));
}
Some(Command::Controls(next)) => {
if connected {
diff --git a/codex-rs/realtime-webrtc/src/session_tests.rs b/codex-rs/realtime-webrtc/src/session_tests.rs
index b3f32e432c..a5afc9b912 100644
--- a/codex-rs/realtime-webrtc/src/session_tests.rs
+++ b/codex-rs/realtime-webrtc/src/session_tests.rs
@@ -180,3 +180,21 @@ fn startup_dispatch_is_ordered_but_acknowledgement_does_not_block_setters() {
);
drop(applied);
}
+
+#[tokio::test]
+async fn startup_completion_preserves_classified_failure() {
+ for failure in [
+ crate::ConnectionError::NegotiationTimedOut,
+ crate::ConnectionError::Failed,
+ ] {
+ let (handle, mut commands, _) = handles();
+ let caller = handle.clone();
+ let thread = std::thread::spawn(move || caller.apply_answer_sdp("synthetic-answer".into()));
+ let Some(Command::Answer(_, complete)) = commands.recv().await else {
+ panic!("answer expected")
+ };
+ complete.send(Err(failure)).unwrap();
+ assert_eq!(thread.join().unwrap(), Err(failure));
+ assert_eq!(handle.take_error(), None);
+ }
+}
diff --git a/codex-rs/tui/Cargo.toml b/codex-rs/tui/Cargo.toml
index 040f44f502..f72429a992 100644
--- a/codex-rs/tui/Cargo.toml
+++ b/codex-rs/tui/Cargo.toml
@@ -53,6 +53,8 @@ codex-models-manager = { workspace = true }
codex-otel = { workspace = true }
codex-plugin = { workspace = true }
codex-protocol = { workspace = true }
+codex-realtime-webrtc = { workspace = true }
+futures = { workspace = true }
codex-rollout = { workspace = true }
codex-sandboxing = { workspace = true }
codex-shell-command = { workspace = true }
diff --git a/codex-rs/tui/src/app.rs b/codex-rs/tui/src/app.rs
index e7140a6533..64e4f4038c 100644
--- a/codex-rs/tui/src/app.rs
+++ b/codex-rs/tui/src/app.rs
@@ -235,6 +235,7 @@ mod pets;
mod platform_actions;
mod plugin_mentions;
mod rate_limit_refresh;
+mod realtime_delivery;
mod recap;
mod reconnect;
mod replay_filter;
@@ -617,6 +618,10 @@ pub(crate) struct App {
windows_sandbox: WindowsSandboxState,
thread_event_channels: HashMap<ThreadId, ThreadEventChannel>,
+ pending_realtime_speech_replay: HashMap<ThreadId, Vec<(String, ThreadItem)>>,
+ pending_realtime_transcript_replay:
+ HashMap<ThreadId, VecDeque<crate::chatwidget::RealtimeTranscriptRecord>>,
+ realtime_replay_order: VecDeque<ThreadId>,
temporary_structured_requests: HashMap<ThreadId, mpsc::UnboundedSender<ServerNotification>>,
/// Track title generation across thread switches and deduplicate automatic requests.
pending_thread_titles: HashSet<(ThreadId, ThreadTitleDestination)>,
diff --git a/codex-rs/tui/src/app/event_dispatch.rs b/codex-rs/tui/src/app/event_dispatch.rs
index cc16390c17..b77998c8f9 100644
--- a/codex-rs/tui/src/app/event_dispatch.rs
+++ b/codex-rs/tui/src/app/event_dispatch.rs
@@ -21,7 +21,7 @@ use codex_app_server_protocol::ThreadGoalStatus;
#[cfg(target_os = "windows")]
use codex_config::types::WindowsSandboxModeToml;
-const SHUTDOWN_FIRST_EXIT_TIMEOUT: Duration = Duration::from_secs(/*secs*/ 2);
+pub(super) const SHUTDOWN_FIRST_EXIT_TIMEOUT: Duration = Duration::from_secs(/*secs*/ 2);
impl App {
pub(super) async fn handle_event(
@@ -785,6 +785,23 @@ impl App {
.apply_reserve_fallback_to_pending_turn(&mut op);
}
let is_user_turn = matches!(&op, AppCommand::UserTurn { .. });
+ let is_realtime_stop = matches!(&op, AppCommand::RealtimeConversationStop { .. });
+ let realtime_stop_thread_id = match &op {
+ AppCommand::RealtimeConversationStop { thread_id } => Some(*thread_id),
+ _ => None,
+ };
+ let realtime_speech_delivery_id = match &op {
+ AppCommand::RealtimeConversationSpeech { delivery_id, .. } => {
+ Some(*delivery_id)
+ }
+ _ => None,
+ };
+ let is_realtime_conversation = matches!(
+ &op,
+ AppCommand::RealtimeConversationStart { .. }
+ | AppCommand::RealtimeConversationStop { .. }
+ | AppCommand::RealtimeConversationSpeech { .. }
+ );
if is_user_turn {
let screen_size = tui.terminal.last_known_screen_size;
self.handle_draw_pre_render(tui, screen_size)?;
@@ -797,6 +814,10 @@ impl App {
}
self.chat_widget.prepare_local_op_submission(&op);
if let Err(err) = self.submit_active_thread_op(app_server, op).await {
+ if let Some(delivery_id) = realtime_speech_delivery_id {
+ self.chat_widget
+ .restore_undelivered_realtime_speech(delivery_id);
+ }
if self.recover_transport_error(&err)
{
return Ok(AppRunControl::Continue);
@@ -817,10 +838,22 @@ impl App {
&& self
.chat_widget
.handle_turn_start_rejection(format!("Failed to start turn: {err:#}"));
- if !handled {
+ if is_realtime_conversation {
+ let message = format!("Voice conversation failed: {err:#}");
+ if is_realtime_stop {
+ if self.chat_widget.thread_id() == realtime_stop_thread_id {
+ self.chat_widget.reset_realtime_conversation();
+ self.chat_widget.add_error_message(message);
+ }
+ } else {
+ self.chat_widget.on_realtime_error(message);
+ }
+ tracing::error!(error = ?err, "realtime conversation request failed");
+ } else if handled {
+ tracing::error!(error = ?err, "failed to start turn through app server");
+ } else {
return Err(err);
}
- tracing::error!(error = ?err, "failed to start turn through app server");
}
}
AppEvent::ConfirmSafetyBufferedRetry {
@@ -1656,6 +1689,47 @@ impl App {
self.sync_active_thread_personality_setting(app_server, personality)
.await;
}
+ AppEvent::RealtimeWebrtcOfferCreated {
+ thread_id,
+ attempt_id,
+ result,
+ } => {
+ if self.chat_widget.thread_id() == Some(thread_id) {
+ self.chat_widget
+ .on_realtime_webrtc_offer_created(thread_id, attempt_id, result);
+ } else if let Ok(offer) = result {
+ offer.handle.close();
+ }
+ }
+ AppEvent::RealtimeWebrtcConnected {
+ thread_id,
+ attempt_id,
+ result,
+ } => {
+ if self.chat_widget.thread_id() == Some(thread_id) {
+ self.chat_widget
+ .on_realtime_webrtc_connected(attempt_id, result);
+ }
+ }
+ AppEvent::StopRealtimeConversation { thread_id } => {
+ match tokio::time::timeout(
+ SHUTDOWN_FIRST_EXIT_TIMEOUT,
+ app_server.thread_realtime_stop(thread_id),
+ )
+ .await
+ {
+ Ok(Ok(())) => {}
+ Ok(Err(error)) => tracing::warn!(
+ %thread_id,
+ %error,
+ "failed to stop voice conversation after switching threads"
+ ),
+ Err(_) => tracing::warn!(
+ %thread_id,
+ "timed out stopping voice conversation after switching threads"
+ ),
+ }
+ }
AppEvent::SettingsSelectionClosed => {
self.app_event_tx.send(AppEvent::SettingsSelectionSettled);
}
@@ -3369,6 +3443,7 @@ impl App {
})
}
ExitMode::Immediate => {
+ self.stop_realtime_conversation(app_server).await;
self.pending_shutdown_exit_thread_id = None;
AppRunControl::Exit(ExitReason::UserRequested)
}
diff --git a/codex-rs/tui/src/app/misalignment_policy.rs b/codex-rs/tui/src/app/misalignment_policy.rs
index 68e4dbed08..3f3dbad870 100644
--- a/codex-rs/tui/src/app/misalignment_policy.rs
+++ b/codex-rs/tui/src/app/misalignment_policy.rs
@@ -4,6 +4,7 @@
use super::*;
use crate::app_server_session::turn_permissions_overrides;
use crate::chatwidget::MisalignmentReview;
+use crate::chatwidget::MisalignmentTurnSource;
use codex_app_server_protocol::ClientRequest;
use codex_app_server_protocol::TurnStartParams;
use codex_app_server_protocol::TurnStartResponse;
@@ -161,6 +162,10 @@ impl App {
Ok(response) => {
let store = &self.ensure_thread_channel(review.thread_id).store;
store.lock().await.active_turn_id = Some(response.turn.id.clone());
+ self.chat_widget.clear_misalignment_for_new_turn(
+ &response.turn.id,
+ MisalignmentTurnSource::AcknowledgedContinuation,
+ );
self.chat_widget.handle_server_notification(
ServerNotification::TurnStarted(
codex_app_server_protocol::TurnStartedNotification {
diff --git a/codex-rs/tui/src/app/realtime_delivery.rs b/codex-rs/tui/src/app/realtime_delivery.rs
new file mode 100644
index 0000000000..c0c1ba9dae
--- /dev/null
+++ b/codex-rs/tui/src/app/realtime_delivery.rs
@@ -0,0 +1,392 @@
+//! Preserve voice answers and accepted transcripts across thread-widget replacement.
+//!
+//! A queued speech command can outlive its widget. Retain its bounded final
+//! item by original thread until replay either renders that item or needs a
+//! fallback, without adding history to whichever thread is currently visible.
+
+use super::*;
+use codex_protocol::models::MessagePhase;
+use std::collections::HashMap;
+use std::collections::HashSet;
+
+type ReplayedVoiceTextCounts = HashMap<(String, String), usize>;
+const MAX_INACTIVE_REALTIME_REPLAY_THREADS: usize = 16;
+
+fn normalized_replay_text(text: &str) -> String {
+ text.split_whitespace().collect::<Vec<_>>().join(" ")
+}
+
+fn record_voice_item(
+ item: &ThreadItem,
+ turn_id: &str,
+ delegated: &mut HashSet<String>,
+ items: &mut HashMap<(String, String, &'static str), String>,
+) {
+ match item {
+ ThreadItem::UserMessage { id, content, .. } => {
+ if let Some(input) = crate::chatwidget::realtime_delegation_input(content) {
+ delegated.insert(turn_id.to_string());
+ let display = crate::chatwidget::realtime_delegation_display_text(input);
+ items.insert(
+ (turn_id.to_string(), id.clone(), "user"),
+ normalized_replay_text(&display),
+ );
+ }
+ }
+ ThreadItem::AgentMessage {
+ id,
+ text,
+ phase: Some(MessagePhase::FinalAnswer) | None,
+ ..
+ } if delegated.contains(turn_id)
+ && !crate::chatwidget::is_private_realtime_agent_item(item)
+ && !text.trim().is_empty() =>
+ {
+ items.insert(
+ (turn_id.to_string(), id.clone(), "assistant"),
+ normalized_replay_text(text),
+ );
+ }
+ _ => {}
+ }
+}
+
+pub(super) fn replayed_voice_texts_from_turns(turns: &[Turn]) -> ReplayedVoiceTextCounts {
+ let mut delegated = HashSet::new();
+ let mut items = HashMap::new();
+ for turn in turns {
+ for item in &turn.items {
+ record_voice_item(item, &turn.id, &mut delegated, &mut items);
+ }
+ }
+ count_replayed_voice_texts(items)
+}
+
+fn count_replayed_voice_texts(
+ items: HashMap<(String, String, &'static str), String>,
+) -> ReplayedVoiceTextCounts {
+ let mut counts = HashMap::new();
+ for ((_, _, role), text) in items {
+ if !text.is_empty() {
+ *counts.entry((role.to_string(), text)).or_insert(0) += 1;
+ }
+ }
+ counts
+}
+
+pub(super) fn replayed_voice_texts(snapshot: &ThreadEventSnapshot) -> ReplayedVoiceTextCounts {
+ let mut delegated = snapshot
+ .delegated_turns
+ .iter()
+ .cloned()
+ .collect::<HashSet<_>>();
+ let mut items = HashMap::new();
+ for turn in &snapshot.turns {
+ for item in &turn.items {
+ record_voice_item(item, &turn.id, &mut delegated, &mut items);
+ }
+ }
+ for event in &snapshot.events {
+ let ThreadBufferedEvent::Notification(notification) = event else {
+ continue;
+ };
+ match notification.as_ref() {
+ ServerNotification::ItemStarted(n) => {
+ if let ThreadItem::UserMessage { content, .. } = &n.item
+ && crate::chatwidget::realtime_delegation_input(content).is_some()
+ {
+ delegated.insert(n.turn_id.clone());
+ }
+ }
+ ServerNotification::ItemCompleted(n) => {
+ record_voice_item(&n.item, &n.turn_id, &mut delegated, &mut items);
+ }
+ ServerNotification::TurnCompleted(n) if n.turn.status == TurnStatus::Completed => {
+ if let Some(item) = n.turn.items.iter().rev().find(|item| {
+ matches!(
+ item,
+ ThreadItem::AgentMessage {
+ phase: Some(MessagePhase::FinalAnswer) | None,
+ ..
+ }
+ ) && !crate::chatwidget::is_private_realtime_agent_item(item)
+ }) {
+ record_voice_item(item, &n.turn.id, &mut delegated, &mut items);
+ }
+ }
+ _ => {}
+ }
+ }
+ count_replayed_voice_texts(items)
+}
+
+impl App {
+ pub(super) fn note_realtime_replay_thread(&mut self, thread_id: ThreadId) {
+ if self.realtime_replay_order.contains(&thread_id) {
+ return;
+ }
+ if self.realtime_replay_order.len() >= MAX_INACTIVE_REALTIME_REPLAY_THREADS
+ && let Some(oldest) = self.realtime_replay_order.pop_front()
+ {
+ self.pending_realtime_transcript_replay.remove(&oldest);
+ self.pending_realtime_speech_replay.remove(&oldest);
+ }
+ self.realtime_replay_order.push_back(thread_id);
+ }
+
+ pub(super) fn forget_realtime_replay_thread(&mut self, thread_id: ThreadId) {
+ self.realtime_replay_order
+ .retain(|saved| *saved != thread_id);
+ self.pending_realtime_transcript_replay.remove(&thread_id);
+ self.pending_realtime_speech_replay.remove(&thread_id);
+ }
+
+ pub(super) fn retain_inactive_realtime_transcript(
+ &mut self,
+ thread_id: ThreadId,
+ notification: &ServerNotification,
+ ) {
+ let (role, text, complete) = match notification {
+ ServerNotification::ThreadRealtimeTranscriptDelta(n) => {
+ (n.role.as_str(), n.delta.as_str(), false)
+ }
+ ServerNotification::ThreadRealtimeTranscriptDone(n) => {
+ (n.role.as_str(), n.text.as_str(), true)
+ }
+ _ => return,
+ };
+ if !matches!(role, "user" | "assistant") {
+ return;
+ }
+ self.note_realtime_replay_thread(thread_id);
+ let records = self
+ .pending_realtime_transcript_replay
+ .entry(thread_id)
+ .or_default();
+ if complete {
+ if text.trim().is_empty() {
+ if let Some(index) = records
+ .iter()
+ .rposition(|record| record.role == role && !record.complete)
+ {
+ records.remove(index);
+ }
+ return;
+ }
+ let mut bounded = text.to_string();
+ if bounded.len() > crate::chatwidget::MAX_TRANSCRIPT_BYTES {
+ let mut end = crate::chatwidget::MAX_TRANSCRIPT_BYTES;
+ while !bounded.is_char_boundary(end) {
+ end -= 1;
+ }
+ bounded.truncate(end);
+ }
+ if let Some(partial) = records
+ .iter_mut()
+ .rev()
+ .find(|record| record.role == role && !record.complete)
+ {
+ partial.text = bounded;
+ partial.complete = true;
+ return;
+ }
+ if records.len() >= crate::chatwidget::MAX_REPLAY_TRANSCRIPT_CELLS {
+ records.pop_front();
+ }
+ records.push_back(crate::chatwidget::RealtimeTranscriptRecord {
+ role: role.to_string(),
+ text: bounded,
+ complete: true,
+ });
+ } else {
+ if text.is_empty() {
+ return;
+ }
+ let partial = records
+ .iter_mut()
+ .rev()
+ .find(|record| record.role == role && !record.complete);
+ if let Some(partial) = partial {
+ partial.text.push_str(text);
+ if partial.text.len() > crate::chatwidget::MAX_TRANSCRIPT_BYTES {
+ let mut start = partial.text.len() - crate::chatwidget::MAX_TRANSCRIPT_BYTES;
+ while !partial.text.is_char_boundary(start) {
+ start += 1;
+ }
+ partial.text.drain(..start);
+ }
+ } else {
+ let mut bounded = text.to_string();
+ if bounded.len() > crate::chatwidget::MAX_TRANSCRIPT_BYTES {
+ let mut start = bounded.len() - crate::chatwidget::MAX_TRANSCRIPT_BYTES;
+ while !bounded.is_char_boundary(start) {
+ start += 1;
+ }
+ bounded.drain(..start);
+ }
+ if records.len() >= crate::chatwidget::MAX_REPLAY_TRANSCRIPT_CELLS {
+ records.pop_front();
+ }
+ records.push_back(crate::chatwidget::RealtimeTranscriptRecord {
+ role: role.to_string(),
+ text: bounded,
+ complete: false,
+ });
+ }
+ }
+ }
+
+ pub(super) fn retain_realtime_replay_state_before_replace(&mut self) {
+ if let Some(thread_id) = self.chat_widget.thread_id() {
+ if self.chat_widget.may_receive_realtime_transcripts() {
+ self.note_realtime_replay_thread(thread_id);
+ // Even a session without captions can receive a final queued caption
+ // after this widget has been replaced.
+ self.pending_realtime_transcript_replay
+ .entry(thread_id)
+ .or_default();
+ }
+ let cells = self.chat_widget.take_realtime_transcript_cells_for_replay();
+ if !cells.is_empty() {
+ self.note_realtime_replay_thread(thread_id);
+ let pending = self
+ .pending_realtime_transcript_replay
+ .entry(thread_id)
+ .or_default();
+ // Only the active widget produces these cells; reattachment consumes them
+ // before another widget for this thread can become active.
+ // Up to 32 accepted cells plus one partial for each speaker.
+ debug_assert!(pending.len() + cells.len() <= 34);
+ pending.extend(cells);
+ }
+ }
+ for (thread_id, turn_id, item) in self
+ .chat_widget
+ .take_undelivered_realtime_speech_for_replay()
+ {
+ self.note_realtime_replay_thread(thread_id);
+ let pending = self
+ .pending_realtime_speech_replay
+ .entry(thread_id)
+ .or_default();
+ // The active widget is the only producer for a thread; selecting
+ // that thread reconciles these records before it can produce more.
+ debug_assert!(pending.len() < 16);
+ pending.push((turn_id, item));
+ }
+ }
+
+ pub(super) fn restore_realtime_replay_state_after_replay(
+ &mut self,
+ replayed_final_items: &HashMap<(String, String), String>,
+ mut replayed_voice_texts: ReplayedVoiceTextCounts,
+ ) {
+ let Some(thread_id) = self.chat_widget.thread_id() else {
+ return;
+ };
+ self.realtime_replay_order
+ .retain(|saved| *saved != thread_id);
+ let mut retained_assistant_captions = HashMap::<String, usize>::new();
+ if let Some(mut cells) = self.pending_realtime_transcript_replay.remove(&thread_id) {
+ cells.retain(|record| {
+ if !record.complete {
+ return true;
+ }
+ let key = (record.role.clone(), normalized_replay_text(&record.text));
+ if let Some(count) = replayed_voice_texts.get_mut(&key)
+ && *count > 0
+ {
+ *count -= 1;
+ return false;
+ }
+ true
+ });
+ for record in &cells {
+ if record.complete && record.role == "assistant" {
+ *retained_assistant_captions
+ .entry(normalized_replay_text(&record.text))
+ .or_default() += 1;
+ }
+ }
+ self.chat_widget.restore_realtime_transcript_cells(cells);
+ }
+ let Some(pending) = self.pending_realtime_speech_replay.remove(&thread_id) else {
+ return;
+ };
+ for (turn_id, item) in pending {
+ let ThreadItem::AgentMessage { id, text, .. } = &item else {
+ continue;
+ };
+ // A matching ID with empty text did not render the saved answer.
+ // Any nonempty snapshot text is authoritative even if it changed.
+ if replayed_final_items
+ .get(&(turn_id.clone(), id.clone()))
+ .is_none_or(|text| text.trim().is_empty())
+ {
+ // A completed caption restored above already displays this
+ // answer. Consume only one exact match; partial or distinct
+ // captions must not hide an undelivered fallback.
+ let spoken_text = text.trim().strip_prefix("[FINAL]").unwrap_or(text);
+ if let Some(count) =
+ retained_assistant_captions.get_mut(&normalized_replay_text(spoken_text))
+ && *count > 0
+ {
+ *count -= 1;
+ continue;
+ }
+ self.chat_widget
+ .render_undelivered_realtime_speech(thread_id, turn_id, item);
+ }
+ }
+ }
+}
+
+pub(super) fn completed_agent_items(
+ snapshot: &ThreadEventSnapshot,
+) -> HashMap<(String, String), String> {
+ let mut completed = completed_agent_items_from_turns(&snapshot.turns);
+ for event in &snapshot.events {
+ let ThreadBufferedEvent::Notification(notification) = event else {
+ continue;
+ };
+ match notification.as_ref() {
+ ServerNotification::ItemCompleted(completed_item) => {
+ if let ThreadItem::AgentMessage { id, text, .. } = &completed_item.item
+ && !text.trim().is_empty()
+ {
+ completed.insert((completed_item.turn_id.clone(), id.clone()), text.clone());
+ }
+ }
+ ServerNotification::TurnCompleted(completed_turn)
+ if completed_turn.turn.status == TurnStatus::Completed =>
+ {
+ for item in &completed_turn.turn.items {
+ if let ThreadItem::AgentMessage { id, text, .. } = item
+ && !text.trim().is_empty()
+ {
+ completed
+ .insert((completed_turn.turn.id.clone(), id.clone()), text.clone());
+ }
+ }
+ }
+ _ => {}
+ }
+ }
+ completed
+}
+
+pub(super) fn completed_agent_items_from_turns(
+ turns: &[Turn],
+) -> HashMap<(String, String), String> {
+ let mut completed = HashMap::new();
+ for turn in turns {
+ for item in &turn.items {
+ if let ThreadItem::AgentMessage { id, text, .. } = item
+ && !text.trim().is_empty()
+ {
+ completed.insert((turn.id.clone(), id.clone()), text.clone());
+ }
+ }
+ }
+ completed
+}
diff --git a/codex-rs/tui/src/app/session_lifecycle.rs b/codex-rs/tui/src/app/session_lifecycle.rs
index 8f2a0d804f..2240d773d5 100644
--- a/codex-rs/tui/src/app/session_lifecycle.rs
+++ b/codex-rs/tui/src/app/session_lifecycle.rs
@@ -484,6 +484,7 @@ impl App {
/// This helper copies every known nickname/role from `AgentNavigationState` into the
/// replacement widget so that replayed collab items render agent names immediately.
pub(super) fn replace_chat_widget(&mut self, mut chat_widget: ChatWidget) {
+ self.retain_realtime_replay_state_before_replace();
chat_widget.cyber_policy_notice = self.chat_widget.cyber_policy_notice.clone();
self.commit_animation = None;
// Transfer the last-written terminal title to the replacement widget
@@ -581,6 +582,16 @@ impl App {
return Ok(());
}
let previous_thread_id = self.active_thread_id;
+ // Notifications already routed to the active channel must reach the old
+ // widget before its transcript state is transferred. Events routed after
+ // deactivation are retained separately for that thread.
+ if let Some(mut receiver) = self.active_thread_rx.take() {
+ while let Ok(event) = receiver.try_recv() {
+ self.handle_thread_event_now_recovering_file_changes(event)
+ .await;
+ }
+ self.active_thread_rx = Some(receiver);
+ }
self.store_active_thread_receiver().await;
self.active_thread_id = None;
let Some((receiver, mut snapshot)) = self.activate_thread_for_replay(thread_id).await
@@ -656,6 +667,10 @@ impl App {
self.recap.seed_from_progress(recap_progress, now);
self.schedule_recap_check(thread_id, now);
+ // The old widget owns the local helper and its undelivered answers.
+ // Transfer replay state before stopping its backend voice session.
+ self.retain_realtime_replay_state_before_replace();
+ self.stop_realtime_conversation(app_server).await;
self.render_thread_snapshot(tui, app_server, thread_id, snapshot, !is_replay_only)?;
if is_replay_only
&& self
@@ -749,6 +764,9 @@ impl App {
pub(super) fn reset_thread_event_state(&mut self) {
self.abort_all_thread_event_listeners();
self.thread_event_channels.clear();
+ self.pending_realtime_speech_replay.clear();
+ self.pending_realtime_transcript_replay.clear();
+ self.realtime_replay_order.clear();
self.pending_server_profiles.clear();
self.agents_overview.activity.clear();
self.agent_navigation.clear();
diff --git a/codex-rs/tui/src/app/side.rs b/codex-rs/tui/src/app/side.rs
index 9dca8bb7f7..db616b3aa2 100644
--- a/codex-rs/tui/src/app/side.rs
+++ b/codex-rs/tui/src/app/side.rs
@@ -524,6 +524,7 @@ impl App {
} else {
self.refresh_pending_thread_approvals().await;
}
+ self.forget_realtime_replay_thread(thread_id);
self.sync_active_agent_label();
}
diff --git a/codex-rs/tui/src/app/startup.rs b/codex-rs/tui/src/app/startup.rs
index 2bb22a51df..13626e677f 100644
--- a/codex-rs/tui/src/app/startup.rs
+++ b/codex-rs/tui/src/app/startup.rs
@@ -707,6 +707,9 @@ See the Codex keymap documentation for supported actions and examples."
pending_shutdown_exit_thread_id: None,
windows_sandbox: WindowsSandboxState::default(),
thread_event_channels: HashMap::new(),
+ pending_realtime_speech_replay: HashMap::new(),
+ pending_realtime_transcript_replay: HashMap::new(),
+ realtime_replay_order: VecDeque::new(),
temporary_structured_requests: HashMap::new(),
pending_thread_titles: HashSet::new(),
thread_event_listener_tasks: HashMap::new(),
diff --git a/codex-rs/tui/src/app/test_support.rs b/codex-rs/tui/src/app/test_support.rs
index f750ff70df..26932ed15c 100644
--- a/codex-rs/tui/src/app/test_support.rs
+++ b/codex-rs/tui/src/app/test_support.rs
@@ -64,6 +64,9 @@ pub(super) async fn make_test_app() -> App {
pending_shutdown_exit_thread_id: None,
windows_sandbox: WindowsSandboxState::default(),
thread_event_channels: HashMap::new(),
+ pending_realtime_speech_replay: HashMap::new(),
+ pending_realtime_transcript_replay: HashMap::new(),
+ realtime_replay_order: VecDeque::new(),
temporary_structured_requests: HashMap::new(),
pending_thread_titles: HashSet::new(),
thread_event_listener_tasks: HashMap::new(),
diff --git a/codex-rs/tui/src/app/tests.rs b/codex-rs/tui/src/app/tests.rs
index bd98ce8a6c..988e979b2a 100644
--- a/codex-rs/tui/src/app/tests.rs
+++ b/codex-rs/tui/src/app/tests.rs
@@ -42,6 +42,10 @@ mod patch_approval_tests;
mod permission_shortcuts_tests;
mod plugin_catalog;
mod rate_limits;
+#[path = "tests/realtime_handoff_e2e.rs"]
+mod realtime_handoff_e2e;
+#[path = "tests/realtime_start.rs"]
+mod realtime_start;
#[path = "tests/recap_generation_tests.rs"]
mod recap_generation;
mod safety_buffering;
@@ -1334,6 +1338,7 @@ async fn replayed_turn_complete_submits_restored_queued_follow_up() {
while new_op_rx.try_recv().is_ok() {}
app.replay_thread_snapshot(
ThreadEventSnapshot {
+ delegated_turns: Vec::new(),
session: None,
turns: Vec::new(),
events: vec![ThreadBufferedEvent::Notification(Box::new(
@@ -1387,6 +1392,7 @@ async fn replay_only_thread_keeps_restored_queue_visible() {
app.replay_thread_snapshot(
ThreadEventSnapshot {
+ delegated_turns: Vec::new(),
session: None,
turns: Vec::new(),
events: vec![ThreadBufferedEvent::Notification(Box::new(
@@ -1438,6 +1444,7 @@ async fn replay_thread_snapshot_keeps_queue_when_running_state_only_comes_from_s
app.replay_thread_snapshot(
ThreadEventSnapshot {
+ delegated_turns: Vec::new(),
session: None,
turns: Vec::new(),
events: vec![],
@@ -1487,6 +1494,7 @@ async fn replay_thread_snapshot_in_progress_turn_restores_running_queue_state()
app.replay_thread_snapshot(
ThreadEventSnapshot {
+ delegated_turns: Vec::new(),
session: None,
turns: vec![test_turn("turn-1", TurnStatus::InProgress, Vec::new())],
events: Vec::new(),
@@ -1516,6 +1524,7 @@ async fn replay_thread_snapshot_in_progress_turn_restores_running_state_without_
app.replay_thread_snapshot(
ThreadEventSnapshot {
+ delegated_turns: Vec::new(),
session: None,
turns: vec![test_turn("turn-1", TurnStatus::InProgress, Vec::new())],
events: Vec::new(),
@@ -1558,6 +1567,7 @@ async fn replay_thread_snapshot_does_not_submit_queue_before_replay_catches_up()
app.replay_thread_snapshot(
ThreadEventSnapshot {
+ delegated_turns: Vec::new(),
session: None,
turns: Vec::new(),
events: vec![
@@ -1699,6 +1709,7 @@ async fn replay_thread_snapshot_restores_collaboration_mode_for_draft_submit() {
app.replay_thread_snapshot(
ThreadEventSnapshot {
+ delegated_turns: Vec::new(),
session: None,
turns: Vec::new(),
events: vec![],
@@ -1779,6 +1790,7 @@ async fn replay_thread_snapshot_restores_collaboration_mode_without_input() {
app.replay_thread_snapshot(
ThreadEventSnapshot {
+ delegated_turns: Vec::new(),
session: None,
turns: Vec::new(),
events: vec![],
@@ -1829,6 +1841,7 @@ async fn replayed_interrupted_turn_restores_queued_input_to_composer() {
app.replay_thread_snapshot(
ThreadEventSnapshot {
+ delegated_turns: Vec::new(),
session: None,
turns: Vec::new(),
events: vec![ThreadBufferedEvent::Notification(Box::new(
@@ -3714,6 +3727,7 @@ async fn replay_snapshot_with_pending_request_suppresses_replay_notices() {
app.replay_thread_snapshot(
ThreadEventSnapshot {
+ delegated_turns: Vec::new(),
session: Some(test_thread_session(thread_id, test_path_buf("/tmp/main"))),
turns: Vec::new(),
events: vec![
@@ -4096,6 +4110,7 @@ async fn replayed_file_change_approval_recovers_snapshot_changes() {
let cwd = test_path_buf("/tmp/project").abs();
app.replay_thread_snapshot(
ThreadEventSnapshot {
+ delegated_turns: Vec::new(),
session: Some(test_thread_session(thread_id, cwd.clone().into_path_buf())),
turns: vec![test_turn(
"turn-replayed-approval",
@@ -5053,6 +5068,7 @@ async fn side_thread_snapshot_does_not_refresh_from_fork_history() {
.insert(side_thread_id, SideThreadState::new(parent_thread_id));
let snapshot = ThreadEventSnapshot {
+ delegated_turns: Vec::new(),
session: Some(ThreadSessionState {
rollout_path: None,
..test_thread_session(side_thread_id, test_path_buf("/tmp/side"))
@@ -5081,6 +5097,7 @@ async fn side_thread_snapshot_skips_session_header_preamble() {
.insert(side_thread_id, SideThreadState::new(parent_thread_id));
let snapshot = ThreadEventSnapshot {
+ delegated_turns: Vec::new(),
session: Some(ThreadSessionState {
forked_from_id: Some(parent_thread_id),
fork_parent_title: None,
@@ -5256,6 +5273,7 @@ async fn active_side_thread_renders_live_mcp_startup_notifications() {
app.activate_thread_channel(side_thread_id).await;
app.replay_thread_snapshot(
ThreadEventSnapshot {
+ delegated_turns: Vec::new(),
session: Some(test_thread_session(
side_thread_id,
test_path_buf("/tmp/side"),
@@ -5839,6 +5857,9 @@ async fn make_test_app() -> App {
pending_shutdown_exit_thread_id: None,
windows_sandbox: WindowsSandboxState::default(),
thread_event_channels: HashMap::new(),
+ pending_realtime_speech_replay: HashMap::new(),
+ pending_realtime_transcript_replay: HashMap::new(),
+ realtime_replay_order: VecDeque::new(),
temporary_structured_requests: HashMap::new(),
pending_thread_titles: HashSet::new(),
thread_event_listener_tasks: HashMap::new(),
@@ -5933,6 +5954,9 @@ pub(super) async fn make_test_app_with_channels() -> (
pending_shutdown_exit_thread_id: None,
windows_sandbox: WindowsSandboxState::default(),
thread_event_channels: HashMap::new(),
+ pending_realtime_speech_replay: HashMap::new(),
+ pending_realtime_transcript_replay: HashMap::new(),
+ realtime_replay_order: VecDeque::new(),
temporary_structured_requests: HashMap::new(),
pending_thread_titles: HashSet::new(),
thread_event_listener_tasks: HashMap::new(),
@@ -8049,6 +8073,7 @@ async fn replay_thread_snapshot_replays_turn_history_in_order() {
let thread_id = ThreadId::new();
app.replay_thread_snapshot(
ThreadEventSnapshot {
+ delegated_turns: Vec::new(),
session: Some(test_thread_session(
thread_id,
test_path_buf("/home/user/project"),
@@ -8165,6 +8190,7 @@ async fn replace_chat_widget_reseeds_collab_agent_metadata_for_replay() {
app.replay_thread_snapshot(
ThreadEventSnapshot {
+ delegated_turns: Vec::new(),
session: None,
turns: Vec::new(),
events: vec![ThreadBufferedEvent::Notification(Box::new(
@@ -8240,6 +8266,7 @@ async fn refreshed_snapshot_session_persists_resumed_turns() {
..initial_session.clone()
};
let mut snapshot = ThreadEventSnapshot {
+ delegated_turns: Vec::new(),
session: Some(initial_session),
turns: Vec::new(),
events: Vec::new(),
diff --git a/codex-rs/tui/src/app/tests/advanced_reasoning_tests.rs b/codex-rs/tui/src/app/tests/advanced_reasoning_tests.rs
index 72f1a7fda3..765278421f 100644
--- a/codex-rs/tui/src/app/tests/advanced_reasoning_tests.rs
+++ b/codex-rs/tui/src/app/tests/advanced_reasoning_tests.rs
@@ -65,6 +65,7 @@ async fn switching_from_ultra_thread_restores_configured_plan_effort() {
app.replay_thread_snapshot(
ThreadEventSnapshot {
+ delegated_turns: Vec::new(),
session: Some(ultra_session),
turns: Vec::new(),
events: Vec::new(),
@@ -74,6 +75,7 @@ async fn switching_from_ultra_thread_restores_configured_plan_effort() {
);
app.replay_thread_snapshot(
ThreadEventSnapshot {
+ delegated_turns: Vec::new(),
session: Some(normal_session),
turns: Vec::new(),
events: Vec::new(),
diff --git a/codex-rs/tui/src/app/tests/disconnect_tests.rs b/codex-rs/tui/src/app/tests/disconnect_tests.rs
index 59bb0fd583..c6117fdfa2 100644
--- a/codex-rs/tui/src/app/tests/disconnect_tests.rs
+++ b/codex-rs/tui/src/app/tests/disconnect_tests.rs
@@ -258,6 +258,38 @@ async fn disconnected_command_center_keeps_input_and_blocks_actions() -> Result<
Ok(())
}
+#[tokio::test]
+async fn remote_disconnect_retires_voice_before_reconnecting() -> Result<()> {
+ let (mut app, _events, _ops) = make_test_app_with_channels().await;
+ let thread_id = ThreadId::new();
+ app.active_thread_id = Some(thread_id);
+ app.chat_widget
+ .handle_thread_session_quiet(test_thread_session(thread_id, app.config.cwd.to_path_buf()));
+ app.app_server_target = AppServerTarget::Remote {
+ endpoint: crate::resolve_remote_addr("ws://127.0.0.1:9")?,
+ };
+ crate::chatwidget::activate_voice_for_thread(&mut app.chat_widget, thread_id);
+ assert!(app.chat_widget.is_current_realtime_attempt(
+ thread_id, /*attempt_id*/ 0, /*input_generation*/ 0
+ ));
+
+ assert!(app.begin_reconnect());
+ assert_eq!(
+ (
+ app.reconnect.offline,
+ app.chat_widget.is_current_realtime_attempt(
+ thread_id, /*attempt_id*/ 0, /*input_generation*/ 0
+ ),
+ ),
+ (true, false)
+ );
+ assert!(app.begin_reconnect());
+ assert!(!app.chat_widget.is_current_realtime_attempt(
+ thread_id, /*attempt_id*/ 0, /*input_generation*/ 0
+ ));
+ Ok(())
+}
+
// Common framing/bootstrap for both transports. Returning None deliberately loses a reply.
pub(super) async fn serve_reconnect_requests<S, F>(
mut socket: tokio_tungstenite::WebSocketStream<S>,
diff --git a/codex-rs/tui/src/app/tests/mcp_startup.rs b/codex-rs/tui/src/app/tests/mcp_startup.rs
index 40d0922099..ffc6db5cc4 100644
--- a/codex-rs/tui/src/app/tests/mcp_startup.rs
+++ b/codex-rs/tui/src/app/tests/mcp_startup.rs
@@ -36,6 +36,7 @@ async fn subagent_mcp_startup_settles_while_cached_servers_remain_deferred() {
app.activate_thread_channel(subagent_thread_id).await;
app.replay_thread_snapshot(
ThreadEventSnapshot {
+ delegated_turns: Vec::new(),
session: Some(test_thread_session(
subagent_thread_id,
test_path_buf("/tmp/subagent"),
diff --git a/codex-rs/tui/src/app/tests/patch_approval_tests.rs b/codex-rs/tui/src/app/tests/patch_approval_tests.rs
index 3bc8ec323a..c01f6789f3 100644
--- a/codex-rs/tui/src/app/tests/patch_approval_tests.rs
+++ b/codex-rs/tui/src/app/tests/patch_approval_tests.rs
@@ -287,6 +287,7 @@ async fn replayed_patch_approval_pager_recovers_stored_turn_changes() {
let cwd = app.chat_widget.config_ref().cwd.to_path_buf();
app.replay_thread_snapshot(
ThreadEventSnapshot {
+ delegated_turns: Vec::new(),
session: Some(test_thread_session(thread_id, cwd)),
turns: vec![test_turn(
TURN_ID,
diff --git a/codex-rs/tui/src/app/tests/realtime_handoff_e2e.rs b/codex-rs/tui/src/app/tests/realtime_handoff_e2e.rs
new file mode 100644
index 0000000000..305650684b
--- /dev/null
+++ b/codex-rs/tui/src/app/tests/realtime_handoff_e2e.rs
@@ -0,0 +1,270 @@
+//! Real Core-to-app-server-to-TUI handoff coverage without an audio device.
+
+use super::*;
+use crate::chatwidget::UserMessage;
+use app_test_support::MockResponsesConfig;
+use app_test_support::create_mock_responses_server_sequence_unchecked;
+use codex_app_server_client::AppServerEvent;
+use codex_app_server_protocol::ServerNotification;
+use codex_login::AuthCredentialsStoreMode;
+use codex_login::AuthKeyringBackendKind;
+use codex_model_provider_info::ModelProviderInfo;
+use core_test_support::responses;
+use core_test_support::responses::WebSocketConnectionConfig;
+use pretty_assertions::assert_eq;
+use ratatui::buffer::Buffer;
+use ratatui::layout::Rect;
+use serde_json::json;
+use std::time::Duration;
+use tokio::time::timeout;
+use wiremock::Mock;
+use wiremock::ResponseTemplate;
+use wiremock::matchers::method;
+use wiremock::matchers::path;
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn delegated_core_events_keep_private_output_hidden_and_deliver_final_speech() -> Result<()> {
+ core_test_support::skip_if_no_network!(Ok(()));
+
+ let mut commentary =
+ responses::ev_assistant_message("private-commentary", "PRIVATE COMMENTARY");
+ commentary["item"]["phase"] = json!("commentary");
+ let mut final_answer = responses::ev_assistant_message(
+ "public-final",
+ "[ANALYSIS] is the marker you asked about.",
+ );
+ final_answer["item"]["phase"] = json!("final_answer");
+ let model_server = create_mock_responses_server_sequence_unchecked(vec![
+ responses::sse(vec![
+ responses::ev_response_created("delegated-response"),
+ responses::ev_reasoning_item("private-reasoning", &["PRIVATE REASONING"], &[]),
+ commentary,
+ final_answer,
+ responses::ev_completed("delegated-response"),
+ ]),
+ responses::sse(vec![
+ responses::ev_response_created("typed-response"),
+ responses::ev_assistant_message("typed-final", "Typed answer stays visible."),
+ responses::ev_completed("typed-response"),
+ ]),
+ ])
+ .await;
+ Mock::given(method("POST"))
+ .and(path("/v1/live"))
+ .respond_with(
+ ResponseTemplate::new(200)
+ .insert_header("Location", "/v1/live/rtc_tui_handoff")
+ .set_body_string("v=answer\r\n"),
+ )
+ .mount(&model_server)
+ .await;
+ let realtime_server =
+ responses::start_websocket_server_with_headers(vec![WebSocketConnectionConfig {
+ requests: vec![
+ vec![
+ json!({
+ "type": "session.started",
+ "session": { "id": "sess_tui_handoff", "instructions": "backend prompt" }
+ }),
+ json!({
+ "type": "delegation.created",
+ "offset_ms": 100,
+ "item": {
+ "id": "delegation_tui",
+ "type": "delegation",
+ "target": "client",
+ "content": [{"type": "input_text", "text": "explain the marker"}]
+ }
+ }),
+ ],
+ vec![],
+ vec![],
+ vec![],
+ ],
+ response_headers: Vec::new(),
+ accept_delay: None,
+ close_after_requests: false,
+ }])
+ .await;
+
+ let (mut app, mut app_events, mut ops) = make_test_app_with_channels().await;
+ let codex_home = tempfile::tempdir()?;
+ MockResponsesConfig::new(&model_server.uri())
+ .with_root_config(&format!(
+ "experimental_realtime_ws_base_url = {:?}\nexperimental_realtime_webrtc_call_base_url = {:?}",
+ realtime_server.uri(),
+ format!("{}/v1", model_server.uri()),
+ ))
+ .with_extra_config("[realtime]\nversion = \"v3\"\ntype = \"conversational\"")
+ .write(codex_home.path())?;
+ codex_login::login_with_api_key(
+ codex_home.path(),
+ "sk-test-key",
+ AuthCredentialsStoreMode::File,
+ AuthKeyringBackendKind::default(),
+ )?;
+ app.config.codex_home = codex_home.path().to_path_buf().abs();
+ app.config.sqlite = codex_state::SqliteConfig::new_for_testing(codex_home.path().abs());
+ app.config.cli_auth_credentials_store_mode = AuthCredentialsStoreMode::File;
+ app.config.model = Some("mock-model".to_string());
+ app.config.model_provider_id = "mock_provider".to_string();
+ app.config.model_provider = ModelProviderInfo {
+ name: "Mock provider for test".to_string(),
+ base_url: Some(format!("{}/v1", model_server.uri())),
+ request_max_retries: Some(0),
+ stream_max_retries: Some(0),
+ ..ModelProviderInfo::default()
+ };
+ app.config.experimental_realtime_ws_base_url = Some(realtime_server.uri().to_string());
+ app.config.experimental_realtime_webrtc_call_base_url =
+ Some(format!("{}/v1", model_server.uri()));
+
+ let mut app_server = Box::pin(crate::start_embedded_app_server_for_picker(&app.config)).await?;
+ let started = app_server.start_thread(&app.config).await?;
+ let thread_id = started.session.thread_id;
+ app.enqueue_primary_thread_session(started.session, started.turns)
+ .await?;
+ crate::chatwidget::activate_voice_for_thread(&mut app.chat_widget, thread_id);
+ while ops.try_recv().is_ok() {}
+ let mut tui = crate::tui::test_support::make_test_tui()?;
+ app_server
+ .thread_realtime_start(thread_id, "v=offer\r\n".to_string(), /*model*/ None)
+ .await?;
+ // The sideband fixture sends its delegation after the first outbound request.
+ app_server
+ .thread_realtime_append_speech(thread_id, "fixture prompt".to_string())
+ .await?;
+
+ let mut rendered = Vec::new();
+ let speech = timeout(Duration::from_secs(15), async {
+ loop {
+ tokio::select! {
+ event = app_server.next_event() => {
+ app.handle_app_server_event(
+ &app_server,
+ event.expect("embedded app-server should stay connected"),
+ ).await;
+ app.drain_active_thread_events(&mut tui).await?;
+ }
+ event = app_events.recv() => {
+ let event = event.expect("TUI event stream should stay open");
+ if let AppEvent::InsertHistoryCell(cell) = &event {
+ rendered.extend(cell.display_lines(/*width*/ 80).into_iter().map(|line| line.to_string()));
+ }
+ app.handle_event(&mut tui, &mut app_server, event).await?;
+ }
+ op = ops.recv() => {
+ let op = op.expect("TUI command stream should stay open");
+ if matches!(op, Op::RealtimeConversationSpeech { .. }) {
+ break Ok::<_, color_eyre::Report>(op);
+ }
+ app.handle_event(&mut tui, &mut app_server, AppEvent::CodexOp(op)).await?;
+ }
+ }
+ }
+ })
+ .await??;
+ assert!(
+ matches!(&speech, Op::RealtimeConversationSpeech { text, .. } if text.as_str() == "[ANALYSIS] is the marker you asked about."),
+ "unexpected speech: {speech:?}"
+ );
+ app.handle_event(&mut tui, &mut app_server, AppEvent::CodexOp(speech))
+ .await?;
+ assert_eq!(
+ timeout(
+ Duration::from_secs(5),
+ realtime_server.wait_for_request(/*connection_index*/ 0, /*request_index*/ 1),
+ )
+ .await?
+ .body_json(),
+ json!({
+ "type": "session.context.append",
+ "content": [{
+ "type": "input_text",
+ "text": "[ANALYSIS] is the marker you asked about."
+ }],
+ "channel": "speakable"
+ })
+ );
+ let rendered = rendered.join("\n");
+ assert!(!rendered.contains("PRIVATE REASONING"), "{rendered}");
+ assert!(!rendered.contains("PRIVATE COMMENTARY"), "{rendered}");
+ let area = Rect::new(
+ /*x*/ 0, /*y*/ 0, /*width*/ 80, /*height*/ 24,
+ );
+ let mut buffer = Buffer::empty(area);
+ app.chat_widget.render(area, &mut buffer);
+ let visible = buffer
+ .content()
+ .iter()
+ .map(ratatui::buffer::Cell::symbol)
+ .collect::<String>();
+ assert!(!visible.contains("PRIVATE REASONING"), "{visible}");
+ assert!(!visible.contains("PRIVATE COMMENTARY"), "{visible}");
+ assert!(
+ ops.try_recv().is_err(),
+ "a final answer must be sent only once"
+ );
+
+ // A typed request on the same live thread must follow the normal visible path.
+ app.chat_widget
+ .restore_user_message_to_composer(UserMessage::from("typed follow-up"));
+ app.chat_widget
+ .handle_key_event(crossterm::event::KeyEvent::new(
+ crossterm::event::KeyCode::Enter,
+ crossterm::event::KeyModifiers::NONE,
+ ));
+ let typed_op = next_user_turn_op(&mut ops);
+ app.handle_event(&mut tui, &mut app_server, AppEvent::CodexOp(typed_op))
+ .await?;
+ let mut typed_rendered = Vec::new();
+ timeout(Duration::from_secs(10), async {
+ loop {
+ tokio::select! {
+ event = app_server.next_event() => {
+ let event = event.expect("embedded app-server should stay connected");
+ let completed = matches!(&event, AppServerEvent::ServerNotification(notification)
+ if matches!(notification.as_ref(), ServerNotification::TurnCompleted(_)));
+ app.handle_app_server_event(&app_server, event).await;
+ app.drain_active_thread_events(&mut tui).await?;
+ if completed {
+ break Ok::<_, color_eyre::Report>(());
+ }
+ }
+ event = app_events.recv() => {
+ let event = event.expect("TUI event stream should stay open");
+ if let AppEvent::InsertHistoryCell(cell) = &event {
+ typed_rendered.extend(cell.display_lines(/*width*/ 80).into_iter().map(|line| line.to_string()));
+ }
+ app.handle_event(&mut tui, &mut app_server, event).await?;
+ }
+ op = ops.recv() => {
+ let op = op.expect("TUI command stream should stay open");
+ assert!(!matches!(op, Op::RealtimeConversationSpeech { .. }), "typed answer must not be spoken");
+ app.handle_event(&mut tui, &mut app_server, AppEvent::CodexOp(op)).await?;
+ }
+ }
+ }
+ })
+ .await??;
+ while let Ok(event) = app_events.try_recv() {
+ if let AppEvent::InsertHistoryCell(cell) = &event {
+ typed_rendered.extend(
+ cell.display_lines(/*width*/ 80)
+ .into_iter()
+ .map(|line| line.to_string()),
+ );
+ }
+ app.handle_event(&mut tui, &mut app_server, event).await?;
+ }
+ assert!(
+ typed_rendered
+ .join("\n")
+ .contains("Typed answer stays visible."),
+ "typed final must appear in history: {typed_rendered:?}"
+ );
+ assert!(ops.try_recv().is_err(), "typed answer must not be spoken");
+ realtime_server.shutdown().await;
+ app_server.shutdown().await?;
+ Ok(())
+}
diff --git a/codex-rs/tui/src/app/tests/realtime_start.rs b/codex-rs/tui/src/app/tests/realtime_start.rs
new file mode 100644
index 0000000000..76c83a20e8
--- /dev/null
+++ b/codex-rs/tui/src/app/tests/realtime_start.rs
@@ -0,0 +1,64 @@
+//! Voice startup rejection coverage without an audio device.
+
+use super::*;
+use crate::app::tests::session_lifecycle_requests::recorded_params;
+use crate::app::tests::session_lifecycle_requests::start_recording_remote_app_server;
+
+#[tokio::test]
+async fn rejected_voice_start_does_not_leave_local_capture_active() -> Result<()> {
+ for replay_only in [false, true] {
+ let (mut app, mut events, _ops) = make_test_app_with_channels().await;
+ let (mut app_server, requests, proxy) =
+ start_recording_remote_app_server(&app.config).await?;
+ let thread_id = ThreadId::new();
+ app.chat_widget
+ .handle_thread_session_quiet(test_thread_session(
+ thread_id,
+ app.config.cwd.to_path_buf(),
+ ));
+ crate::chatwidget::activate_voice_for_thread(&mut app.chat_widget, thread_id);
+ assert!(app.chat_widget.may_receive_realtime_transcripts());
+ if replay_only {
+ app.app_server_target = AppServerTarget::Remote {
+ endpoint: crate::resolve_remote_addr("ws://127.0.0.1:9")?,
+ };
+ app.active_thread_id = Some(thread_id);
+ app.ensure_thread_channel(thread_id).mark_replay_only();
+ assert!(app.thread_unavailable(thread_id));
+ }
+ let mut tui = crate::tui::test_support::make_test_tui()?;
+ Box::pin(app.handle_event(
+ &mut tui,
+ &mut app_server,
+ AppEvent::CodexOp(Op::RealtimeConversationStart {
+ thread_id,
+ offer_sdp: String::from("v=0\r\n").into(),
+ }),
+ ))
+ .await?;
+
+ assert!(!app.chat_widget.may_receive_realtime_transcripts());
+ assert!(recorded_params(&requests, "thread/realtime/start").is_empty());
+ let rendered = std::iter::from_fn(|| events.try_recv().ok())
+ .filter_map(|event| match event {
+ AppEvent::InsertHistoryCell(cell) => Some(
+ cell.display_lines(/*width*/ 80)
+ .into_iter()
+ .map(|line| line.to_string())
+ .collect::<Vec<_>>()
+ .join("\n"),
+ ),
+ _ => None,
+ })
+ .collect::<Vec<_>>()
+ .join("\n");
+ if replay_only {
+ insta::assert_snapshot!("rejected_voice_start_replay_only", rendered);
+ } else {
+ insta::assert_snapshot!("rejected_voice_start_no_active_thread", rendered);
+ }
+ app_server.shutdown().await?;
+ proxy.await??;
+ }
+ Ok(())
+}
diff --git a/codex-rs/tui/src/app/tests/snapshots/codex_tui__app__tests__realtime_start__rejected_voice_start_no_active_thread.snap b/codex-rs/tui/src/app/tests/snapshots/codex_tui__app__tests__realtime_start__rejected_voice_start_no_active_thread.snap
new file mode 100644
index 0000000000..dcab8f663d
--- /dev/null
+++ b/codex-rs/tui/src/app/tests/snapshots/codex_tui__app__tests__realtime_start__rejected_voice_start_no_active_thread.snap
@@ -0,0 +1,6 @@
+---
+source: tui/src/app/tests/realtime_start.rs
+assertion_line: 59
+expression: rendered
+---
+■ No active thread is available.
diff --git a/codex-rs/tui/src/app/tests/snapshots/codex_tui__app__tests__realtime_start__rejected_voice_start_replay_only.snap b/codex-rs/tui/src/app/tests/snapshots/codex_tui__app__tests__realtime_start__rejected_voice_start_replay_only.snap
new file mode 100644
index 0000000000..fb035f0336
--- /dev/null
+++ b/codex-rs/tui/src/app/tests/snapshots/codex_tui__app__tests__realtime_start__rejected_voice_start_replay_only.snap
@@ -0,0 +1,6 @@
+---
+source: tui/src/app/tests/realtime_start.rs
+assertion_line: 57
+expression: rendered
+---
+■ This conversation is read-only or unavailable; no operation was sent.
diff --git a/codex-rs/tui/src/app/tests/startup.rs b/codex-rs/tui/src/app/tests/startup.rs
index 02b3be4e6b..7b822b6df4 100644
--- a/codex-rs/tui/src/app/tests/startup.rs
+++ b/codex-rs/tui/src/app/tests/startup.rs
@@ -1406,6 +1406,7 @@ async fn ignore_same_thread_resume_allows_retrying_read_only_view() -> Result<()
&app_server,
thread_id,
ThreadEventSnapshot {
+ delegated_turns: Vec::new(),
session: Some(session),
turns: vec![test_turn("running", TurnStatus::InProgress, Vec::new())],
events: Vec::new(),
diff --git a/codex-rs/tui/src/app/thread_events.rs b/codex-rs/tui/src/app/thread_events.rs
index e192656dad..f577a1f02a 100644
--- a/codex-rs/tui/src/app/thread_events.rs
+++ b/codex-rs/tui/src/app/thread_events.rs
@@ -11,6 +11,7 @@ use std::borrow::Cow;
#[derive(Debug, Clone)]
pub(super) struct ThreadEventSnapshot {
pub(super) session: Option<ThreadSessionState>,
+ pub(super) delegated_turns: Vec<String>,
pub(super) turns: Vec<Turn>,
pub(super) events: Vec<ThreadBufferedEvent>,
pub(super) input_state: Option<ThreadInputState>,
@@ -24,6 +25,24 @@ pub(super) enum ThreadBufferedEvent {
FeedbackSubmission(FeedbackThreadEvent),
}
+fn is_voice_handoff_item(item: &ThreadItem) -> bool {
+ matches!(item, ThreadItem::UserMessage { content, .. }
+ if crate::chatwidget::realtime_delegation_input(content).is_some())
+}
+
+fn hide_private_items_after_voice_handoff(items: &mut Vec<ThreadItem>, delegated: bool) {
+ let has_marker = items.iter().any(is_voice_handoff_item);
+ // An evicted marker leaves no reliable order, so hide private items
+ // conservatively. A present marker lets earlier typed output survive.
+ let mut voice_started = delegated && !has_marker;
+ items.retain(|item| {
+ if is_voice_handoff_item(item) {
+ voice_started = true;
+ }
+ !voice_started || !crate::chatwidget::is_private_realtime_agent_item(item)
+ });
+}
+
#[derive(Debug, Clone, PartialEq, Eq)]
pub(super) struct FeedbackThreadEvent {
pub(super) category: FeedbackCategory,
@@ -53,6 +72,7 @@ pub(super) struct ThreadEventStore {
pub(super) capacity: usize,
pub(super) active: bool,
pub(super) buffered_agent_message_delta_bytes: usize,
+ delegated_turns: VecDeque<String>,
recap_progress: recap::RecapProgress,
}
@@ -95,6 +115,7 @@ impl ThreadEventStore {
capacity,
active: false,
buffered_agent_message_delta_bytes: 0,
+ delegated_turns: VecDeque::new(),
recap_progress: recap::RecapProgress::default(),
}
}
@@ -147,6 +168,18 @@ impl ThreadEventStore {
}
fn push_notification_inner(&mut self, notification: Cow<'_, ServerNotification>) {
+ let user_item = match notification.as_ref() {
+ ServerNotification::ItemStarted(n) => Some((&n.turn_id, &n.item)),
+ ServerNotification::ItemCompleted(n) => Some((&n.turn_id, &n.item)),
+ _ => None,
+ };
+ if let Some((turn_id, ThreadItem::UserMessage { content, .. })) = user_item
+ && turn_id.len() <= 512
+ && crate::chatwidget::realtime_delegation_input(content).is_some()
+ && !self.delegated_turns.contains(turn_id)
+ {
+ self.delegated_turns.push_back(turn_id.clone());
+ }
self.pending_interactive_replay
.note_server_notification(notification.as_ref());
match notification.as_ref() {
@@ -204,6 +237,27 @@ impl ThreadEventStore {
}
self.push_replay_notification(notification);
+ self.trim_delegated_turns();
+ }
+
+ fn trim_delegated_turns(&mut self) {
+ while self.delegated_turns.len() > self.capacity {
+ // Keep an evicted user marker while this turn still has buffered output.
+ let stale = self.delegated_turns.iter().position(|turn_id| {
+ !self.buffer.iter().any(|event| {
+ matches!(event, ThreadBufferedEvent::Notification(n) if match n.as_ref() {
+ ServerNotification::ItemCompleted(item) => &item.turn_id == turn_id,
+ ServerNotification::AgentMessageDelta(delta) => &delta.turn_id == turn_id,
+ ServerNotification::ReasoningSummaryTextDelta(delta) => &delta.turn_id == turn_id,
+ ServerNotification::ReasoningTextDelta(delta) => &delta.turn_id == turn_id,
+ ServerNotification::ReasoningSummaryPartAdded(part) => &part.turn_id == turn_id,
+ ServerNotification::TurnCompleted(turn) => &turn.turn.id == turn_id,
+ _ => false,
+ })
+ })
+ });
+ self.delegated_turns.remove(stale.unwrap_or(0));
+ }
}
pub(super) fn push_request(&mut self, request: ServerRequest) {
@@ -242,6 +296,7 @@ impl ThreadEventStore {
pub(super) fn snapshot(&self) -> ThreadEventSnapshot {
let mut snapshot = ThreadEventSnapshot {
session: self.session.clone(),
+ delegated_turns: self.delegated_turns.iter().cloned().collect(),
turns: self.turns.clone(),
// Thread switches replay buffered events into a rebuilt ChatWidget. Only replay
// interactive prompts that are still pending, or answered approvals/input will reappear.
@@ -263,6 +318,99 @@ impl ThreadEventStore {
if let Some(latest_turn_id) = &self.latest_turn_id {
replay_filter::omit_resolved_misalignment_errors(&mut snapshot, latest_turn_id);
}
+ let delegated = snapshot
+ .turns
+ .iter()
+ .filter(|turn| {
+ turn.items.iter().any(|item| {
+ matches!(item, ThreadItem::UserMessage { content, .. }
+ if crate::chatwidget::realtime_delegation_input(content).is_some())
+ })
+ })
+ .map(|turn| turn.id.clone())
+ .chain(self.delegated_turns.iter().cloned())
+ .collect::<std::collections::HashSet<_>>();
+ for turn in &mut snapshot.turns {
+ hide_private_items_after_voice_handoff(&mut turn.items, delegated.contains(&turn.id));
+ }
+ let buffered_markers = snapshot
+ .events
+ .iter()
+ .filter_map(|event| match event {
+ ThreadBufferedEvent::Notification(notification) => match notification.as_ref() {
+ ServerNotification::ItemStarted(n) if is_voice_handoff_item(&n.item) => {
+ Some(n.turn_id.clone())
+ }
+ ServerNotification::ItemCompleted(n) if is_voice_handoff_item(&n.item) => {
+ Some(n.turn_id.clone())
+ }
+ _ => None,
+ },
+ _ => None,
+ })
+ .collect::<std::collections::HashSet<_>>();
+ let mut voice_started = delegated
+ .difference(&buffered_markers)
+ .cloned()
+ .collect::<std::collections::HashSet<_>>();
+ let mut typed_items = std::collections::HashSet::new();
+ snapshot.events.retain_mut(|event| {
+ let ThreadBufferedEvent::Notification(notification) = event else {
+ return true;
+ };
+ match notification.as_ref() {
+ ServerNotification::ItemStarted(n) if is_voice_handoff_item(&n.item) => {
+ voice_started.insert(n.turn_id.clone());
+ }
+ ServerNotification::ItemCompleted(n) if is_voice_handoff_item(&n.item) => {
+ voice_started.insert(n.turn_id.clone());
+ }
+ ServerNotification::ItemStarted(n) => {
+ if let ThreadItem::AgentMessage { id, .. } | ThreadItem::Reasoning { id, .. } =
+ &n.item
+ {
+ let key = (n.turn_id.clone(), id.clone());
+ if voice_started.contains(&n.turn_id) {
+ typed_items.remove(&key);
+ } else {
+ typed_items.insert(key);
+ }
+ }
+ }
+ _ => {}
+ }
+ match notification.as_mut() {
+ ServerNotification::AgentMessageDelta(n) => {
+ !voice_started.contains(&n.turn_id)
+ || typed_items.contains(&(n.turn_id.clone(), n.item_id.clone()))
+ }
+ ServerNotification::ReasoningSummaryTextDelta(n) => {
+ !voice_started.contains(&n.turn_id)
+ || typed_items.contains(&(n.turn_id.clone(), n.item_id.clone()))
+ }
+ ServerNotification::ReasoningTextDelta(n) => {
+ !voice_started.contains(&n.turn_id)
+ || typed_items.contains(&(n.turn_id.clone(), n.item_id.clone()))
+ }
+ ServerNotification::ReasoningSummaryPartAdded(n) => {
+ !voice_started.contains(&n.turn_id)
+ || typed_items.contains(&(n.turn_id.clone(), n.item_id.clone()))
+ }
+ ServerNotification::ItemCompleted(n) if voice_started.contains(&n.turn_id) => {
+ matches!(&n.item, ThreadItem::AgentMessage { id, .. } | ThreadItem::Reasoning { id, .. }
+ if typed_items.contains(&(n.turn_id.clone(), id.clone())))
+ || !crate::chatwidget::is_private_realtime_agent_item(&n.item)
+ }
+ ServerNotification::TurnCompleted(n) if delegated.contains(&n.turn.id) => {
+ hide_private_items_after_voice_handoff(
+ &mut n.turn.items,
+ /*delegated*/ true,
+ );
+ true
+ }
+ _ => true,
+ }
+ });
snapshot
}
@@ -430,12 +578,16 @@ mod tests {
use codex_app_server_protocol::HookRunSummary as AppServerHookRunSummary;
use codex_app_server_protocol::HookScope as AppServerHookScope;
use codex_app_server_protocol::HookStartedNotification;
+ use codex_app_server_protocol::ItemCompletedNotification;
+ use codex_app_server_protocol::ItemStartedNotification;
use codex_app_server_protocol::McpToolCallProgressNotification;
+ use codex_app_server_protocol::ReasoningSummaryTextDeltaNotification;
use codex_app_server_protocol::RequestId as AppServerRequestId;
use codex_app_server_protocol::ThreadRealtimeAudioChunk;
use codex_app_server_protocol::ThreadRealtimeOutputAudioDeltaNotification;
use codex_app_server_protocol::TurnCompletedNotification;
use codex_app_server_protocol::TurnStartedNotification;
+ use codex_app_server_protocol::UserInput;
use codex_config::types::ApprovalsReviewer;
use codex_protocol::models::PermissionProfile;
use pretty_assertions::assert_eq;
@@ -818,4 +970,93 @@ mod tests {
serde_json::to_value(notification).expect("MCP notification should serialize"),
);
}
+
+ #[test]
+ fn buffered_voice_handoff_retains_only_preexisting_typed_reasoning() {
+ let thread_id = ThreadId::new().to_string();
+ let turn_id = "mixed-reasoning-turn".to_string();
+ let mut store = ThreadEventStore::new(/*capacity*/ 16);
+ let typed = ThreadItem::Reasoning {
+ id: "typed-reasoning".into(),
+ summary: Vec::new(),
+ content: Vec::new(),
+ };
+ store.push_notification(ServerNotification::ItemStarted(ItemStartedNotification {
+ thread_id: thread_id.clone(),
+ turn_id: turn_id.clone(),
+ item: typed,
+ started_at_ms: 0,
+ }));
+ store.push_notification(ServerNotification::ItemStarted(ItemStartedNotification {
+ thread_id: thread_id.clone(),
+ turn_id: turn_id.clone(),
+ item: ThreadItem::UserMessage {
+ id: "voice-marker".into(),
+ client_id: None,
+ content: vec![UserInput::Text {
+ text:
+ "<realtime_delegation><input>spoken follow-up</input></realtime_delegation>"
+ .into(),
+ text_elements: Vec::new(),
+ }],
+ },
+ started_at_ms: 0,
+ }));
+ for (id, text) in [
+ ("typed-reasoning", "Typed tail"),
+ ("private-reasoning", "Private voice reasoning"),
+ ] {
+ if id == "private-reasoning" {
+ store.push_notification(ServerNotification::ItemStarted(ItemStartedNotification {
+ thread_id: thread_id.clone(),
+ turn_id: turn_id.clone(),
+ item: ThreadItem::Reasoning {
+ id: id.into(),
+ summary: Vec::new(),
+ content: Vec::new(),
+ },
+ started_at_ms: 0,
+ }));
+ }
+ store.push_notification(ServerNotification::ReasoningSummaryTextDelta(
+ ReasoningSummaryTextDeltaNotification {
+ thread_id: thread_id.clone(),
+ turn_id: turn_id.clone(),
+ item_id: id.into(),
+ delta: text.into(),
+ summary_index: 0,
+ },
+ ));
+ store.push_notification(ServerNotification::ItemCompleted(
+ ItemCompletedNotification {
+ thread_id: thread_id.clone(),
+ turn_id: turn_id.clone(),
+ item: ThreadItem::Reasoning {
+ id: id.into(),
+ summary: vec![text.into()],
+ content: Vec::new(),
+ },
+ completed_at_ms: 0,
+ },
+ ));
+ }
+
+ let retained = store
+ .snapshot()
+ .events
+ .into_iter()
+ .filter_map(|event| match event {
+ ThreadBufferedEvent::Notification(notification) => match *notification {
+ ServerNotification::ReasoningSummaryTextDelta(n) => Some(n.delta),
+ ServerNotification::ItemCompleted(n) => match n.item {
+ ThreadItem::Reasoning { summary, .. } => Some(summary.join("")),
+ _ => None,
+ },
+ _ => None,
+ },
+ _ => None,
+ })
+ .collect::<Vec<_>>();
+ assert_eq!(retained, vec!["Typed tail", "Typed tail"]);
+ }
}
diff --git a/codex-rs/tui/src/app/thread_routing.rs b/codex-rs/tui/src/app/thread_routing.rs
index e688317558..45b370a8bd 100644
--- a/codex-rs/tui/src/app/thread_routing.rs
+++ b/codex-rs/tui/src/app/thread_routing.rs
@@ -13,8 +13,32 @@ use codex_app_server_protocol::TurnInterruptParams;
use codex_app_server_protocol::TurnInterruptResponse;
use codex_app_server_protocol::WarningNotification;
+// Leave time for side-thread cleanup and unsubscribe inside the two-second exit budget.
+const REALTIME_STOP_TIMEOUT: Duration = Duration::from_secs(/*secs*/ 1);
+
impl App {
+ pub(super) async fn stop_realtime_conversation(&mut self, app_server: &mut AppServerSession) {
+ let Some(thread_id) = self.chat_widget.reset_realtime_conversation() else {
+ return;
+ };
+ match tokio::time::timeout(
+ REALTIME_STOP_TIMEOUT,
+ app_server.thread_realtime_stop(thread_id),
+ )
+ .await
+ {
+ Ok(Ok(())) => {}
+ Ok(Err(error)) => {
+ tracing::warn!(%thread_id, %error, "failed to stop voice conversation");
+ }
+ Err(_) => {
+ tracing::warn!(%thread_id, "timed out stopping voice conversation");
+ }
+ }
+ }
+
pub(super) async fn shutdown_current_thread(&mut self, app_server: &mut AppServerSession) {
+ self.stop_realtime_conversation(app_server).await;
self.shutdown_side_threads(app_server).await;
if let Some(thread_id) = self.chat_widget.thread_id() {
if let Err(err) = app_server.thread_unsubscribe(thread_id).await {
@@ -446,6 +470,15 @@ impl App {
op: AppCommand,
) -> Result<()> {
let Some(thread_id) = self.active_thread_id else {
+ if let AppCommand::RealtimeConversationSpeech { delivery_id, .. } = &op {
+ self.chat_widget
+ .restore_undelivered_realtime_speech(*delivery_id);
+ }
+ if let AppCommand::RealtimeConversationStart { thread_id, .. } = &op
+ && self.chat_widget.thread_id() == Some(*thread_id)
+ {
+ self.chat_widget.reset_realtime_conversation();
+ }
self.chat_widget
.add_error_message("No active thread is available.".to_string());
return Ok(());
@@ -461,12 +494,25 @@ impl App {
op: AppCommand,
) -> Result<()> {
if self.thread_unavailable(thread_id) {
+ if let AppCommand::RealtimeConversationSpeech { delivery_id, .. } = &op {
+ self.chat_widget
+ .restore_undelivered_realtime_speech(*delivery_id);
+ }
+ if let AppCommand::RealtimeConversationStart { thread_id, .. } = &op
+ && self.chat_widget.thread_id() == Some(*thread_id)
+ {
+ self.chat_widget.reset_realtime_conversation();
+ }
self.chat_widget.add_error_message(
"This conversation is read-only or unavailable; no operation was sent.".into(),
);
return Ok(());
}
if self.chat_widget.rejects_misalignment_policy_op(&op) {
+ if let AppCommand::RealtimeConversationSpeech { delivery_id, .. } = &op {
+ self.chat_widget
+ .restore_undelivered_realtime_speech(*delivery_id);
+ }
return Ok(());
}
@@ -867,6 +913,60 @@ impl App {
.await?;
Ok(true)
}
+ AppCommand::RealtimeConversationStart {
+ thread_id: realtime_thread_id,
+ offer_sdp,
+ } => {
+ if *realtime_thread_id != thread_id {
+ return Ok(true);
+ }
+ let model = self
+ .chat_widget
+ .config_ref()
+ .experimental_realtime_ws_model
+ .clone();
+ app_server
+ .thread_realtime_start(
+ *realtime_thread_id,
+ String::from(offer_sdp.clone()),
+ model,
+ )
+ .await?;
+ Ok(true)
+ }
+ AppCommand::RealtimeConversationStop {
+ thread_id: realtime_thread_id,
+ } => {
+ app_server.thread_realtime_stop(*realtime_thread_id).await?;
+ Ok(true)
+ }
+ AppCommand::RealtimeConversationSpeech {
+ thread_id: realtime_thread_id,
+ attempt_id,
+ input_generation,
+ delivery_id,
+ text,
+ } => {
+ if !self.chat_widget.has_pending_realtime_speech(*delivery_id) {
+ return Ok(true);
+ }
+ if *realtime_thread_id != thread_id
+ || !self.chat_widget.is_current_realtime_attempt(
+ *realtime_thread_id,
+ *attempt_id,
+ *input_generation,
+ )
+ {
+ self.chat_widget
+ .restore_undelivered_realtime_speech(*delivery_id);
+ return Ok(true);
+ }
+ app_server
+ .thread_realtime_append_speech(*realtime_thread_id, text.as_str().to_owned())
+ .await?;
+ self.chat_widget.accept_realtime_speech(*delivery_id);
+ Ok(true)
+ }
AppCommand::RunUserShellCommand { command } => {
app_server
.thread_shell_command(thread_id, command.to_string())
@@ -1136,6 +1236,7 @@ impl App {
guard.push_notification_ref(&notification);
Some(notification)
} else {
+ self.retain_inactive_realtime_transcript(thread_id, &notification);
guard.push_notification(notification);
None
};
@@ -1423,6 +1524,8 @@ impl App {
}
}
let should_buffer_initial_replay = !turns.is_empty();
+ let replayed_final_items = realtime_delivery::completed_agent_items_from_turns(&turns);
+ let replayed_voice_texts = realtime_delivery::replayed_voice_texts_from_turns(&turns);
if should_buffer_initial_replay {
self.app_event_tx
.send(AppEvent::BeginInitialHistoryReplayBuffer);
@@ -1438,6 +1541,10 @@ impl App {
self.app_event_tx
.send(AppEvent::EndInitialHistoryReplayBuffer);
}
+ self.restore_realtime_replay_state_after_replay(
+ &replayed_final_items,
+ replayed_voice_texts,
+ );
if matches!(presentation, ThreadAttachPresentation::PromptEdit) {
self.chat_widget.emit_prompt_edit_thread_event();
}
@@ -1651,6 +1758,8 @@ impl App {
mut snapshot: ThreadEventSnapshot,
resume_restored_queue: bool,
) {
+ let replayed_final_items = realtime_delivery::completed_agent_items(&snapshot);
+ let replayed_voice_texts = realtime_delivery::replayed_voice_texts(&snapshot);
replay_filter::omit_completed_agent_deltas(&mut snapshot.events);
let request_changes = snapshot
.events
@@ -1694,6 +1803,10 @@ impl App {
self.chat_widget.handle_thread_session(session);
}
}
+ for turn_id in &snapshot.delegated_turns {
+ self.chat_widget
+ .remember_realtime_delegated_reasoning_turn(turn_id);
+ }
let recovered_input = snapshot
.input_state
.as_ref()
@@ -1728,6 +1841,10 @@ impl App {
if recovered_input.is_some() {
self.chat_widget.restore_reconnected_input(recovered_input);
}
+ self.restore_realtime_replay_state_after_replay(
+ &replayed_final_items,
+ replayed_voice_texts,
+ );
self.chat_widget
.set_queue_autosend_suppressed(/*suppressed*/ false);
self.chat_widget
diff --git a/codex-rs/tui/src/app_command.rs b/codex-rs/tui/src/app_command.rs
index f54b00497d..82bc948942 100644
--- a/codex-rs/tui/src/app_command.rs
+++ b/codex-rs/tui/src/app_command.rs
@@ -1,3 +1,4 @@
+use std::fmt;
use std::path::PathBuf;
use codex_app_server_protocol::AskForApproval;
@@ -9,6 +10,7 @@ use codex_app_server_protocol::ReviewTarget;
use codex_app_server_protocol::ToolRequestUserInputResponse;
use codex_app_server_protocol::UserInput;
use codex_config::types::ApprovalsReviewer;
+use codex_protocol::ThreadId;
use codex_protocol::approvals::GuardianAssessmentEvent;
use codex_protocol::config_types::CollaborationMode;
use codex_protocol::config_types::Personality;
@@ -19,13 +21,98 @@ use codex_protocol::models::PermissionProfile;
use codex_protocol::openai_models::ReasoningEffort as ReasoningEffortConfig;
use codex_protocol::request_permissions::RequestPermissionsResponse;
use serde::Serialize;
+use serde::Serializer;
use serde_json::Value;
+/// Keeps ICE credentials out of command diagnostics and session recordings.
+/// Convert back to a string only when sending the offer to app-server signaling.
+#[derive(Clone, PartialEq, Eq)]
+pub(crate) struct RealtimeOfferSdp(String);
+
+impl From<String> for RealtimeOfferSdp {
+ fn from(value: String) -> Self {
+ Self(value)
+ }
+}
+
+impl From<RealtimeOfferSdp> for String {
+ fn from(value: RealtimeOfferSdp) -> Self {
+ value.0
+ }
+}
+
+impl fmt::Debug for RealtimeOfferSdp {
+ fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
+ formatter.write_str("<redacted>")
+ }
+}
+
+impl Serialize for RealtimeOfferSdp {
+ fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+ where
+ S: Serializer,
+ {
+ serializer.serialize_str("<redacted>")
+ }
+}
+
+/// Keeps spoken answers out of command diagnostics and session recordings.
+/// Convert back to a string only at the app-server signaling boundary.
+#[derive(Clone, PartialEq, Eq)]
+pub(crate) struct RealtimeSpeechText(String);
+
+impl From<String> for RealtimeSpeechText {
+ fn from(value: String) -> Self {
+ Self(value)
+ }
+}
+
+impl From<RealtimeSpeechText> for String {
+ fn from(value: RealtimeSpeechText) -> Self {
+ value.0
+ }
+}
+
+impl RealtimeSpeechText {
+ pub(crate) fn as_str(&self) -> &str {
+ &self.0
+ }
+}
+
+impl fmt::Debug for RealtimeSpeechText {
+ fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
+ formatter.write_str("<redacted>")
+ }
+}
+
+impl Serialize for RealtimeSpeechText {
+ fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+ where
+ S: Serializer,
+ {
+ serializer.serialize_str("<redacted>")
+ }
+}
+
#[allow(clippy::large_enum_variant)]
#[derive(Debug, Clone, PartialEq, Serialize)]
pub(crate) enum AppCommand {
Interrupt,
CleanBackgroundTerminals,
+ RealtimeConversationStart {
+ thread_id: ThreadId,
+ offer_sdp: RealtimeOfferSdp,
+ },
+ RealtimeConversationStop {
+ thread_id: ThreadId,
+ },
+ RealtimeConversationSpeech {
+ thread_id: ThreadId,
+ attempt_id: u64,
+ input_generation: u64,
+ delivery_id: u64,
+ text: RealtimeSpeechText,
+ },
RunUserShellCommand {
command: String,
},
@@ -252,3 +339,7 @@ impl From<&AppCommand> for AppCommand {
value.clone()
}
}
+
+#[cfg(test)]
+#[path = "app_command_tests.rs"]
+mod tests;
diff --git a/codex-rs/tui/src/app_command_tests.rs b/codex-rs/tui/src/app_command_tests.rs
new file mode 100644
index 0000000000..c765205c62
--- /dev/null
+++ b/codex-rs/tui/src/app_command_tests.rs
@@ -0,0 +1,70 @@
+//! Protect connection credentials in diagnostics without changing signaling data.
+
+use super::AppCommand;
+use codex_protocol::ThreadId;
+use pretty_assertions::assert_eq;
+use serde_json::json;
+
+#[test]
+fn realtime_offer_is_redacted_in_diagnostics_and_preserved_for_signaling() {
+ let thread_id =
+ ThreadId::from_string("00000000-0000-0000-0000-000000000001").expect("valid thread id");
+ let original_offer =
+ "v=0\r\na=ice-ufrag:private-user\r\na=ice-pwd:private-credential\r\n".to_string();
+ let command = AppCommand::RealtimeConversationStart {
+ thread_id,
+ offer_sdp: original_offer.clone().into(),
+ };
+
+ assert_eq!(
+ format!("{command:?}"),
+ format!("RealtimeConversationStart {{ thread_id: {thread_id:?}, offer_sdp: <redacted> }}")
+ );
+ assert_eq!(
+ serde_json::to_value(&command).expect("command diagnostics serialize"),
+ json!({
+ "RealtimeConversationStart": {
+ "thread_id": thread_id,
+ "offer_sdp": "<redacted>"
+ }
+ })
+ );
+
+ let AppCommand::RealtimeConversationStart { offer_sdp, .. } = command else {
+ panic!("expected the start command");
+ };
+ assert_eq!(String::from(offer_sdp), original_offer);
+}
+
+#[test]
+fn realtime_speech_is_redacted_in_diagnostics_and_preserved_for_signaling() {
+ let thread_id =
+ ThreadId::from_string("00000000-0000-0000-0000-000000000001").expect("valid thread id");
+ let original_text = "Private spoken answer".to_string();
+ let command = AppCommand::RealtimeConversationSpeech {
+ thread_id,
+ attempt_id: 1,
+ input_generation: 2,
+ delivery_id: 3,
+ text: original_text.clone().into(),
+ };
+
+ assert!(!format!("{command:?}").contains(&original_text));
+ assert_eq!(
+ serde_json::to_value(&command).expect("command diagnostics serialize"),
+ json!({
+ "RealtimeConversationSpeech": {
+ "thread_id": thread_id,
+ "attempt_id": 1,
+ "input_generation": 2,
+ "delivery_id": 3,
+ "text": "<redacted>"
+ }
+ })
+ );
+
+ let AppCommand::RealtimeConversationSpeech { text, .. } = command else {
+ panic!("expected the speech command");
+ };
+ assert_eq!(String::from(text), original_text);
+}
diff --git a/codex-rs/tui/src/app_event.rs b/codex-rs/tui/src/app_event.rs
index 1df5a1a7f4..285b7de6d7 100644
--- a/codex-rs/tui/src/app_event.rs
+++ b/codex-rs/tui/src/app_event.rs
@@ -63,6 +63,7 @@ use codex_plugin::PluginCapabilitySummary;
use codex_protocol::config_types::CollaborationModeMask;
use codex_protocol::config_types::Personality;
use codex_protocol::models::ActivePermissionProfile;
+use codex_realtime_webrtc::StartedRealtimeWebrtcSession;
use crate::history_cell::HistoryCell;
@@ -1053,6 +1054,25 @@ pub(crate) enum AppEvent {
/// Update the current personality in the running app and widget.
UpdatePersonality(Personality),
+ /// Result of creating a TUI-owned WebRTC offer for an active thread.
+ RealtimeWebrtcOfferCreated {
+ thread_id: ThreadId,
+ attempt_id: u64,
+ result: Result<StartedRealtimeWebrtcSession, String>,
+ },
+
+ /// Result of establishing the WebRTC connection for an active voice attempt.
+ RealtimeWebrtcConnected {
+ thread_id: ThreadId,
+ attempt_id: u64,
+ result: Result<(), codex_realtime_webrtc::ConnectionError>,
+ },
+
+ /// Stop voice on its original thread after its chat widget is replaced.
+ StopRealtimeConversation {
+ thread_id: ThreadId,
+ },
+
/// Finish a settings selection after its preceding update events have been applied.
SettingsSelectionClosed,
/// Run after any nested settings events emitted while handling the close event.
diff --git a/codex-rs/tui/src/app_server_session.rs b/codex-rs/tui/src/app_server_session.rs
index 9583524099..c5b74996e8 100644
--- a/codex-rs/tui/src/app_server_session.rs
+++ b/codex-rs/tui/src/app_server_session.rs
@@ -6,6 +6,7 @@
mod fs;
mod history;
mod models;
+mod realtime;
mod rollout_history;
mod thread_list;
diff --git a/codex-rs/tui/src/app_server_session/realtime.rs b/codex-rs/tui/src/app_server_session/realtime.rs
new file mode 100644
index 0000000000..f9fff2c00b
--- /dev/null
+++ b/codex-rs/tui/src/app_server_session/realtime.rs
@@ -0,0 +1,91 @@
+//! App-server signaling for TUI-owned realtime WebRTC sessions.
+
+use super::AppServerSession;
+use codex_app_server_protocol::ClientRequest;
+use codex_app_server_protocol::ThreadRealtimeAppendSpeechParams;
+use codex_app_server_protocol::ThreadRealtimeAppendSpeechResponse;
+use codex_app_server_protocol::ThreadRealtimeStartParams;
+use codex_app_server_protocol::ThreadRealtimeStartResponse;
+use codex_app_server_protocol::ThreadRealtimeStartTransport;
+use codex_app_server_protocol::ThreadRealtimeStopParams;
+use codex_app_server_protocol::ThreadRealtimeStopResponse;
+use codex_protocol::ThreadId;
+use codex_protocol::protocol::RealtimeConversationVersion;
+use codex_protocol::protocol::RealtimeOutputModality;
+use color_eyre::eyre::Result;
+use color_eyre::eyre::WrapErr;
+
+impl AppServerSession {
+ pub(crate) async fn thread_realtime_start(
+ &mut self,
+ thread_id: ThreadId,
+ offer_sdp: String,
+ model: Option<String>,
+ ) -> Result<()> {
+ let request_id = self.next_request_id();
+ let _: ThreadRealtimeStartResponse = self
+ .client
+ .request_typed(ClientRequest::ThreadRealtimeStart {
+ request_id,
+ params: ThreadRealtimeStartParams {
+ thread_id: thread_id.to_string(),
+ client_managed_handoffs: Some(true),
+ delegation_ack_filler: None,
+ flush_transcript_tail_on_session_end: None,
+ codex_responses_as_items: None,
+ codex_response_item_prefix: None,
+ codex_response_handoff_mode: None,
+ codex_response_handoff_channel_prefixes: None,
+ model,
+ output_modality: RealtimeOutputModality::Audio,
+ include_startup_context: Some(false),
+ initial_items: None,
+ realtime_start_instructions: None,
+ realtime_end_instructions: None,
+ prompt: None,
+ realtime_session_id: None,
+ transport: Some(ThreadRealtimeStartTransport::Webrtc { sdp: offer_sdp }),
+ version: Some(RealtimeConversationVersion::V3),
+ voice: None,
+ },
+ })
+ .await
+ .wrap_err("thread/realtime/start failed in TUI")?;
+ Ok(())
+ }
+
+ pub(crate) async fn thread_realtime_stop(&mut self, thread_id: ThreadId) -> Result<()> {
+ let request_id = self.next_request_id();
+ let _: ThreadRealtimeStopResponse = self
+ .client
+ .request_typed(ClientRequest::ThreadRealtimeStop {
+ request_id,
+ params: ThreadRealtimeStopParams {
+ thread_id: thread_id.to_string(),
+ },
+ })
+ .await
+ .wrap_err("thread/realtime/stop failed in TUI")?;
+ Ok(())
+ }
+
+ pub(crate) async fn thread_realtime_append_speech(
+ &mut self,
+ thread_id: ThreadId,
+ text: String,
+ ) -> Result<()> {
+ let request_id = self.next_request_id();
+ let _: ThreadRealtimeAppendSpeechResponse = self
+ .client
+ .request_typed(ClientRequest::ThreadRealtimeAppendSpeech {
+ request_id,
+ params: ThreadRealtimeAppendSpeechParams {
+ thread_id: thread_id.to_string(),
+ text,
+ },
+ })
+ .await
+ .wrap_err("thread/realtime/appendSpeech failed in TUI")?;
+ Ok(())
+ }
+}
diff --git a/codex-rs/tui/src/bottom_pane/chat_composer.rs b/codex-rs/tui/src/bottom_pane/chat_composer.rs
index 967c712fe9..6d084e067f 100644
--- a/codex-rs/tui/src/bottom_pane/chat_composer.rs
+++ b/codex-rs/tui/src/bottom_pane/chat_composer.rs
@@ -554,6 +554,7 @@ pub(crate) struct ChatComposer {
mentions_v2_enabled: bool,
goal_command_enabled: bool,
personality_command_enabled: bool,
+ voice_command_enabled: bool,
worktrees_enabled: bool,
windows_degraded_sandbox_active: bool,
side_conversation_active: bool,
@@ -718,6 +719,7 @@ impl ChatComposer {
mentions_v2_enabled: false,
goal_command_enabled: false,
personality_command_enabled: false,
+ voice_command_enabled: false,
worktrees_enabled: false,
windows_degraded_sandbox_active: false,
side_conversation_active: false,
@@ -924,6 +926,10 @@ impl ChatComposer {
self.goal_command_enabled = enabled;
}
+ pub fn set_voice_command_enabled(&mut self, enabled: bool) {
+ self.voice_command_enabled = enabled;
+ }
+
/// Replace composer, editor, and footer-hint key bindings from one runtime snapshot.
///
/// Submit and queue bindings are cached here because composer dispatch must
@@ -1035,19 +1041,11 @@ impl ChatComposer {
.unwrap_or_else(|| footer_height(&footer_props));
let footer_spacing = Self::footer_spacing(footer_hint_height);
let footer_total_height = footer_hint_height + footer_spacing;
- let popup_constraint = match &self.popups.active {
- ActivePopup::Command(popup) => {
- Constraint::Max(popup.calculate_required_height(area.width))
- }
- ActivePopup::File(popup) => Constraint::Max(popup.calculate_required_height()),
- ActivePopup::Skill(popup) => {
- Constraint::Max(popup.calculate_required_height(area.width))
- }
- ActivePopup::MentionV2(popup) => {
- Constraint::Max(popup.calculate_required_height(area.width))
- }
- ActivePopup::None => Constraint::Max(footer_total_height),
- };
+ let popup_constraint = Constraint::Max(
+ self.popups
+ .active
+ .required_height(area.width, footer_total_height),
+ );
let [composer_rect, popup_rect] =
Layout::vertical([Constraint::Min(3), popup_constraint]).areas(area);
let mut textarea_rect = composer_rect.inset(Insets::tlbr(
@@ -4656,13 +4654,10 @@ impl ChatComposer {
+ remote_images_height
+ remote_images_separator
+ 2
- + match &self.popups.active {
- ActivePopup::None => footer_total_height,
- ActivePopup::Command(c) => c.calculate_required_height(width),
- ActivePopup::File(c) => c.calculate_required_height(),
- ActivePopup::Skill(c) => c.calculate_required_height(width),
- ActivePopup::MentionV2(c) => c.calculate_required_height(width),
- }
+ + self
+ .popups
+ .active
+ .required_height(width, footer_total_height)
}
}
diff --git a/codex-rs/tui/src/bottom_pane/chat_composer/popup_state.rs b/codex-rs/tui/src/bottom_pane/chat_composer/popup_state.rs
index 2f97ecda52..da2dfd908b 100644
--- a/codex-rs/tui/src/bottom_pane/chat_composer/popup_state.rs
+++ b/codex-rs/tui/src/bottom_pane/chat_composer/popup_state.rs
@@ -112,6 +112,18 @@ pub(super) enum ActivePopup {
MentionV2(MentionV2Popup),
}
+impl ActivePopup {
+ pub(super) fn required_height(&self, width: u16, footer_total_height: u16) -> u16 {
+ match self {
+ Self::None => footer_total_height,
+ Self::Command(popup) => popup.calculate_required_height(width),
+ Self::File(popup) => popup.calculate_required_height(),
+ Self::Skill(popup) => popup.calculate_required_height(width),
+ Self::MentionV2(popup) => popup.calculate_required_height(width),
+ }
+ }
+}
+
#[cfg(test)]
#[path = "popup_state_tests.rs"]
mod tests;
diff --git a/codex-rs/tui/src/bottom_pane/chat_composer/slash_input.rs b/codex-rs/tui/src/bottom_pane/chat_composer/slash_input.rs
index 22fce854f4..8e4ff2f87b 100644
--- a/codex-rs/tui/src/bottom_pane/chat_composer/slash_input.rs
+++ b/codex-rs/tui/src/bottom_pane/chat_composer/slash_input.rs
@@ -178,6 +178,7 @@ impl<'a> SlashInput<'a> {
service_tier_commands_enabled: self.command_flags.service_tier_commands_enabled,
goal_command_enabled: self.command_flags.goal_command_enabled,
personality_command_enabled: self.command_flags.personality_command_enabled,
+ voice_command_enabled: self.command_flags.voice_command_enabled,
worktrees_enabled: self.command_flags.worktrees_enabled,
windows_degraded_sandbox_active: self.command_flags.allow_elevate_sandbox,
side_conversation_active: self.command_flags.side_conversation_active,
@@ -216,6 +217,7 @@ impl ChatComposer {
service_tier_commands_enabled: self.service_tier_commands_enabled,
goal_command_enabled: self.goal_command_enabled,
personality_command_enabled: self.personality_command_enabled,
+ voice_command_enabled: self.voice_command_enabled,
worktrees_enabled: self.worktrees_enabled,
allow_elevate_sandbox: self.windows_degraded_sandbox_active,
side_conversation_active: self.side_conversation_active,
diff --git a/codex-rs/tui/src/bottom_pane/command_popup.rs b/codex-rs/tui/src/bottom_pane/command_popup.rs
index 81cc8013d8..f577205eba 100644
--- a/codex-rs/tui/src/bottom_pane/command_popup.rs
+++ b/codex-rs/tui/src/bottom_pane/command_popup.rs
@@ -48,6 +48,7 @@ pub(crate) struct CommandPopupFlags {
pub(crate) service_tier_commands_enabled: bool,
pub(crate) goal_command_enabled: bool,
pub(crate) personality_command_enabled: bool,
+ pub(crate) voice_command_enabled: bool,
pub(crate) worktrees_enabled: bool,
pub(crate) windows_degraded_sandbox_active: bool,
pub(crate) side_conversation_active: bool,
@@ -63,6 +64,7 @@ impl From<CommandPopupFlags> for BuiltinCommandFlags {
service_tier_commands_enabled: value.service_tier_commands_enabled,
goal_command_enabled: value.goal_command_enabled,
personality_command_enabled: value.personality_command_enabled,
+ voice_command_enabled: value.voice_command_enabled,
worktrees_enabled: value.worktrees_enabled,
allow_elevate_sandbox: value.windows_degraded_sandbox_active,
side_conversation_active: value.side_conversation_active,
@@ -412,6 +414,31 @@ mod tests {
insta::assert_snapshot!("command_popup_app", format!("{buf:?}"));
}
+ #[cfg(target_os = "macos")]
+ #[test]
+ fn voice_command_popup_snapshot() {
+ let mut popup = CommandPopup::new(
+ CommandPopupFlags {
+ voice_command_enabled: true,
+ ..CommandPopupFlags::default()
+ },
+ Vec::new(),
+ );
+ popup.on_composer_text_change("/voice".to_string());
+
+ let width = 72;
+ let area = Rect::new(
+ /*x*/ 0,
+ /*y*/ 0,
+ width,
+ popup.calculate_required_height(width),
+ );
+ let mut buf = Buffer::empty(area);
+ popup.render_ref(area, &mut buf);
+
+ insta::assert_snapshot!("command_popup_voice", format!("{buf:?}"));
+ }
+
#[cfg(target_os = "macos")]
#[test]
fn default_command_popup_items_snapshot() {
@@ -537,6 +564,7 @@ mod tests {
service_tier_commands_enabled: false,
goal_command_enabled: false,
personality_command_enabled: true,
+ voice_command_enabled: false,
worktrees_enabled: true,
windows_degraded_sandbox_active: false,
side_conversation_active: false,
@@ -565,6 +593,7 @@ mod tests {
service_tier_commands_enabled: false,
goal_command_enabled: false,
personality_command_enabled: false,
+ voice_command_enabled: false,
worktrees_enabled: false,
windows_degraded_sandbox_active: false,
side_conversation_active: false,
@@ -598,6 +627,7 @@ mod tests {
service_tier_commands_enabled: false,
goal_command_enabled: false,
personality_command_enabled: true,
+ voice_command_enabled: false,
worktrees_enabled: true,
windows_degraded_sandbox_active: false,
side_conversation_active: false,
diff --git a/codex-rs/tui/src/bottom_pane/footer.rs b/codex-rs/tui/src/bottom_pane/footer.rs
index 997866be7a..d5cbd09067 100644
--- a/codex-rs/tui/src/bottom_pane/footer.rs
+++ b/codex-rs/tui/src/bottom_pane/footer.rs
@@ -875,9 +875,17 @@ pub(crate) fn footer_hint_items_line(items: &[(String, String)]) -> Line<'static
for (idx, (key, label)) in items.iter().enumerate() {
spans.push(" ".into());
spans.push(key.clone().bold());
- spans.push(format!(" {label}").into());
+ if idx == 0
+ && key == "voice"
+ && let Some(label) = label.strip_prefix("● ")
+ {
+ spans.push(" ●".red());
+ spans.push(format!(" {label}").into());
+ } else {
+ spans.push(format!(" {label}").into());
+ }
if idx + 1 != items.len() {
- spans.push(" ".into());
+ spans.push((if items[0].0 == "voice" { " " } else { " " }).into());
}
}
Line::from(spans)
@@ -1313,6 +1321,43 @@ mod tests {
use ratatui::backend::Backend;
use ratatui::backend::TestBackend;
+ #[test]
+ fn voice_live_microphone_indicator_is_red() {
+ let line = footer_hint_items_line(&[("voice".into(), "● listen".into())]);
+ assert_eq!(line.spans[2].style.fg, Some(ratatui::style::Color::Red));
+ }
+
+ #[test]
+ fn voice_footer_rendering_preserves_text_and_styles() {
+ let items = [
+ ("voice".into(), "● listen".into()),
+ ("ctrl+m".into(), "mute".into()),
+ ];
+ let mut terminal =
+ Terminal::new(TestBackend::new(/*width*/ 40, /*height*/ 1)).expect("create terminal");
+ terminal
+ .draw(|frame| render_footer_hint_items(frame.area(), frame.buffer_mut(), &items))
+ .expect("render voice footer");
+ let backend = terminal.backend();
+ let mut previous_style = None;
+ let style_runs = (0..40)
+ .filter_map(|x| {
+ let cell = backend.buffer().cell((x, 0))?;
+ let style = cell.style();
+ if previous_style == Some(style) {
+ None
+ } else {
+ previous_style = Some(style);
+ Some((x, style))
+ }
+ })
+ .collect::<Vec<_>>();
+ insta::assert_debug_snapshot!(
+ "voice_footer_rendered_styles",
+ (backend.to_string(), style_runs)
+ );
+ }
+
fn snapshot_footer(name: &str, props: FooterProps) {
snapshot_footer_with_mode_indicator(
name, /*width*/ 80, &props, /*collaboration_mode_indicator*/ None,
diff --git a/codex-rs/tui/src/bottom_pane/mod.rs b/codex-rs/tui/src/bottom_pane/mod.rs
index f249fad8b2..00712baab9 100644
--- a/codex-rs/tui/src/bottom_pane/mod.rs
+++ b/codex-rs/tui/src/bottom_pane/mod.rs
@@ -546,6 +546,11 @@ impl BottomPane {
self.request_redraw();
}
+ pub fn set_voice_command_enabled(&mut self, enabled: bool) {
+ self.composer.set_voice_command_enabled(enabled);
+ self.request_redraw();
+ }
+
pub(crate) fn set_side_conversation_active(&mut self, active: bool) {
self.composer.set_side_conversation_active(active);
self.request_redraw();
diff --git a/codex-rs/tui/src/bottom_pane/slash_commands.rs b/codex-rs/tui/src/bottom_pane/slash_commands.rs
index 61393648e7..8c478b3e31 100644
--- a/codex-rs/tui/src/bottom_pane/slash_commands.rs
+++ b/codex-rs/tui/src/bottom_pane/slash_commands.rs
@@ -62,6 +62,7 @@ pub(crate) struct BuiltinCommandFlags {
pub(crate) service_tier_commands_enabled: bool,
pub(crate) goal_command_enabled: bool,
pub(crate) personality_command_enabled: bool,
+ pub(crate) voice_command_enabled: bool,
pub(crate) worktrees_enabled: bool,
pub(crate) allow_elevate_sandbox: bool,
pub(crate) side_conversation_active: bool,
@@ -79,6 +80,7 @@ pub(crate) fn builtins_for_input(flags: BuiltinCommandFlags) -> Vec<(&'static st
.filter(|(_, cmd)| flags.goal_command_enabled || *cmd != SlashCommand::Goal)
.filter(|(_, cmd)| flags.worktrees_enabled || *cmd != SlashCommand::Worktree)
.filter(|(_, cmd)| flags.personality_command_enabled || *cmd != SlashCommand::Personality)
+ .filter(|(_, cmd)| flags.voice_command_enabled || *cmd != SlashCommand::Voice)
.filter(|(_, cmd)| !flags.side_conversation_active || cmd.available_in_side_conversation())
.collect()
}
@@ -173,6 +175,7 @@ mod tests {
service_tier_commands_enabled: true,
goal_command_enabled: true,
personality_command_enabled: true,
+ voice_command_enabled: true,
worktrees_enabled: true,
allow_elevate_sandbox: true,
side_conversation_active: false,
@@ -282,6 +285,13 @@ mod tests {
assert_eq!(find_builtin_command("goal", flags), None);
}
+ #[test]
+ fn voice_command_is_hidden_when_disabled() {
+ let mut flags = all_enabled_flags();
+ flags.voice_command_enabled = false;
+ assert_eq!(find_builtin_command("voice", flags), None);
+ }
+
#[test]
fn usage_command_is_hidden_from_input_when_account_token_activity_is_disabled() {
let mut flags = all_enabled_flags();
diff --git a/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__command_popup__tests__command_popup_voice.snap b/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__command_popup__tests__command_popup_voice.snap
new file mode 100644
index 0000000000..417c3b62eb
--- /dev/null
+++ b/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__command_popup__tests__command_popup_voice.snap
@@ -0,0 +1,15 @@
+---
+source: tui/src/bottom_pane/command_popup.rs
+expression: "format!(\"{buf:?}\")"
+---
+Buffer {
+ area: Rect { x: 0, y: 0, width: 72, height: 1 },
+ content: [
+ " /voice start or stop a live voice conversation ",
+ ],
+ styles: [
+ x: 0, y: 0, fg: Reset, bg: Reset, underline: Reset, modifier: NONE,
+ x: 2, y: 0, fg: Cyan, bg: Reset, underline: Reset, modifier: BOLD,
+ x: 49, y: 0, fg: Reset, bg: Reset, underline: Reset, modifier: NONE,
+ ]
+}
diff --git a/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__footer__tests__voice_footer_rendered_styles.snap b/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__footer__tests__voice_footer_rendered_styles.snap
new file mode 100644
index 0000000000..d71648f1d6
--- /dev/null
+++ b/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__footer__tests__voice_footer_rendered_styles.snap
@@ -0,0 +1,34 @@
+---
+source: tui/src/bottom_pane/footer.rs
+assertion_line: 1355
+expression: "(backend.to_string(), style_runs)"
+---
+(
+ "\" voice ● listen ctrl+m mute \"\n",
+ [
+ (
+ 0,
+ Style::new().fg(Color::Reset).bg(Color::Reset).underline_color(Color::Reset),
+ ),
+ (
+ 3,
+ Style::new().fg(Color::Reset).bg(Color::Reset).underline_color(Color::Reset).bold(),
+ ),
+ (
+ 8,
+ Style::new().red().bg(Color::Reset).underline_color(Color::Reset),
+ ),
+ (
+ 10,
+ Style::new().fg(Color::Reset).bg(Color::Reset).underline_color(Color::Reset),
+ ),
+ (
+ 20,
+ Style::new().fg(Color::Reset).bg(Color::Reset).underline_color(Color::Reset).bold(),
+ ),
+ (
+ 26,
+ Style::new().fg(Color::Reset).bg(Color::Reset).underline_color(Color::Reset),
+ ),
+ ],
+)
diff --git a/codex-rs/tui/src/chatwidget.rs b/codex-rs/tui/src/chatwidget.rs
index afaa3fc953..628e49b352 100644
--- a/codex-rs/tui/src/chatwidget.rs
+++ b/codex-rs/tui/src/chatwidget.rs
@@ -357,6 +357,7 @@ mod mcp_startup;
use self::mcp_startup::McpStartupStatus;
mod misalignment_policy;
pub(crate) use misalignment_policy::MisalignmentReview;
+pub(crate) use misalignment_policy::MisalignmentTurnSource;
mod pets;
mod session_flow;
mod session_header;
@@ -406,7 +407,17 @@ mod recap;
mod reset_credits;
pub(crate) use self::rate_limits::limit_label_for_window;
mod completion;
+mod realtime;
+pub(crate) use realtime::MAX_REPLAY_TRANSCRIPT_CELLS;
+pub(crate) use realtime::MAX_TRANSCRIPT_BYTES;
+pub(crate) use realtime::RealtimeTranscriptRecord;
+pub(crate) use realtime::is_private_realtime_agent_item;
+pub(crate) use realtime::realtime_delegation_display_text;
+pub(crate) use realtime::realtime_delegation_input;
+#[cfg(test)]
+pub(crate) use realtime::tests::activate_voice_for_thread;
mod reasoning_shortcuts;
+use self::realtime::RealtimeConversationUiState;
mod rendering;
mod replay;
mod review;
@@ -631,6 +642,8 @@ pub(crate) struct ChatWidget {
last_unified_wait: Option<UnifiedExecWaitState>,
unified_exec_wait_streak: Option<UnifiedExecWaitStreak>,
turn_lifecycle: TurnLifecycleState,
+ realtime_conversation: RealtimeConversationUiState,
+ realtime_conversation_available_for_thread: bool,
safety_buffering: SafetyBufferingState,
task_complete_pending: bool,
unified_exec_processes: Vec<UnifiedExecProcessSummary>,
@@ -1179,6 +1192,8 @@ impl ChatWidget {
self.update_due_hook_visibility();
self.schedule_hook_timer_if_needed();
self.bottom_pane.pre_draw_tick();
+ self.flush_realtime_transcript_history();
+ self.refresh_realtime_microphone_level();
if let Some(pet) = self.ambient_pet.as_ref() {
pet.schedule_next_frame();
}
@@ -1295,7 +1310,20 @@ impl ChatWidget {
items: &[UserInput],
client_id: Option<&str>,
from_replay: bool,
+ turn_id: &str,
) {
+ if let Some(input) = realtime::realtime_delegation_input(items) {
+ if !from_replay && self.should_hide_realtime_delegation(turn_id) {
+ return;
+ }
+ let input = realtime::realtime_delegation_display_text(input);
+ let projected = [UserInput::Text {
+ text: input,
+ text_elements: Vec::new(),
+ }];
+ self.on_committed_user_message(&projected, client_id, from_replay, turn_id);
+ return;
+ }
let display = Self::user_message_display_from_inputs(items);
if from_replay {
if self.review.is_review_mode {
@@ -1920,9 +1948,14 @@ impl ChatWidget {
/// the main viewport updates.
pub(crate) fn active_cell_transcript_key(&self) -> Option<ActiveCellTranscriptKey> {
let cell = self.transcript.active_cell.as_ref();
+ let realtime_cell = self.realtime_conversation.live_transcript_cell.as_ref();
let token_activity_cell = self.pending_token_activity_output();
let rate_limit_reset_hint = self.pending_rate_limit_reset_hint();
- if cell.is_none() && token_activity_cell.is_none() && rate_limit_reset_hint.is_none() {
+ if cell.is_none()
+ && realtime_cell.is_none()
+ && token_activity_cell.is_none()
+ && rate_limit_reset_hint.is_none()
+ {
return None;
}
Some(ActiveCellTranscriptKey {
@@ -1948,6 +1981,13 @@ impl ChatWidget {
if let Some(cell) = self.transcript.active_cell.as_ref() {
lines.extend(cell.transcript_hyperlink_lines(width));
}
+ if let Some(cell) = self.realtime_conversation.live_transcript_cell.as_ref() {
+ let realtime_lines = cell.transcript_hyperlink_lines(width);
+ if !realtime_lines.is_empty() && !lines.is_empty() {
+ lines.push(HyperlinkLine::from(""));
+ }
+ lines.extend(realtime_lines);
+ }
if let Some(token_activity_cell) = self.pending_token_activity_output() {
let token_activity_lines = token_activity_cell.transcript_hyperlink_lines(width);
if !token_activity_lines.is_empty() && !lines.is_empty() {
@@ -1998,6 +2038,13 @@ fn has_websocket_timing_metrics(summary: RuntimeMetricsSummary) -> bool {
impl Drop for ChatWidget {
fn drop(&mut self) {
+ if self.realtime_conversation.handle.is_some()
+ && let Some(thread_id) = self.thread_id
+ {
+ self.app_event_tx
+ .send(AppEvent::StopRealtimeConversation { thread_id });
+ }
+ self.reset_realtime_conversation();
self.stop_rate_limit_poller();
}
}
diff --git a/codex-rs/tui/src/chatwidget/constructor.rs b/codex-rs/tui/src/chatwidget/constructor.rs
index f45ae82fef..7be8282a2e 100644
--- a/codex-rs/tui/src/chatwidget/constructor.rs
+++ b/codex-rs/tui/src/chatwidget/constructor.rs
@@ -175,6 +175,8 @@ impl ChatWidget {
last_unified_wait: None,
unified_exec_wait_streak: None,
turn_lifecycle: TurnLifecycleState::new(prevent_idle_sleep),
+ realtime_conversation: RealtimeConversationUiState::default(),
+ realtime_conversation_available_for_thread: false,
safety_buffering: SafetyBufferingState::default(),
task_complete_pending: false,
unified_exec_processes: Vec::new(),
@@ -291,6 +293,9 @@ impl ChatWidget {
widget.sync_worktrees_enabled();
widget.sync_plugins_command_enabled();
widget.sync_goal_command_enabled();
+ widget
+ .bottom_pane
+ .set_voice_command_enabled(/*enabled*/ false);
widget.sync_mentions_v2_enabled();
widget
.bottom_pane
diff --git a/codex-rs/tui/src/chatwidget/input_submission.rs b/codex-rs/tui/src/chatwidget/input_submission.rs
index 2abcb5655d..b0ba5c7d1e 100644
--- a/codex-rs/tui/src/chatwidget/input_submission.rs
+++ b/codex-rs/tui/src/chatwidget/input_submission.rs
@@ -410,6 +410,7 @@ impl ChatWidget {
return (false, None);
}
self.dismiss_backend_banner_for_new_turn();
+ self.note_realtime_typed_input(&submitted_message.text);
if render_in_history {
self.input_queue.user_turn_pending_start = true;
}
diff --git a/codex-rs/tui/src/chatwidget/misalignment_policy.rs b/codex-rs/tui/src/chatwidget/misalignment_policy.rs
index fcba6416b9..1be4ed3307 100644
--- a/codex-rs/tui/src/chatwidget/misalignment_policy.rs
+++ b/codex-rs/tui/src/chatwidget/misalignment_policy.rs
@@ -26,6 +26,13 @@ impl MisalignmentReview {
pub(super) struct MisalignmentViolation {
turn_id: Option<String>,
review: Option<Arc<MisalignmentReview>>,
+ retired_voice: bool,
+}
+
+#[derive(Clone, Copy, Eq, PartialEq)]
+pub(crate) enum MisalignmentTurnSource {
+ ServerNotification,
+ AcknowledgedContinuation,
}
const MISALIGNMENT_POLICY_TITLE: &str = "Chat stopped as a precaution";
@@ -41,6 +48,7 @@ impl ChatWidget {
&& !matches!(
op,
AppCommand::Interrupt
+ | AppCommand::RealtimeConversationStop { .. }
| AppCommand::CleanBackgroundTerminals
| AppCommand::OverrideTurnContext { .. }
| AppCommand::ReloadUserConfig
@@ -80,6 +88,10 @@ impl ChatWidget {
{
return;
}
+ let retired_voice = self.realtime_conversation_is_running();
+ if retired_voice {
+ self.stop_realtime_conversation();
+ }
let review = self.thread_id.zip(turn_id.clone()).zip(details).map(
|((thread_id, turn_id), details)| {
Arc::new(MisalignmentReview {
@@ -89,7 +101,11 @@ impl ChatWidget {
})
},
);
- self.misalignment_policy_violation = Some(MisalignmentViolation { turn_id, review });
+ self.misalignment_policy_violation = Some(MisalignmentViolation {
+ turn_id,
+ review,
+ retired_voice,
+ });
self.input_queue.clear();
self.finalize_turn();
self.refresh_pending_input_preview();
@@ -211,11 +227,19 @@ impl ChatWidget {
});
}
- pub(crate) fn clear_misalignment_for_new_turn(&mut self, turn_id: &str) {
+ pub(crate) fn clear_misalignment_for_new_turn(
+ &mut self,
+ turn_id: &str,
+ source: MisalignmentTurnSource,
+ ) {
if self
.misalignment_policy_violation
.as_ref()
- .is_some_and(|violation| violation.turn_id.as_ref().is_some_and(|id| id != turn_id))
+ .is_some_and(|violation| {
+ violation.turn_id.as_ref().is_some_and(|id| id != turn_id)
+ && (!violation.retired_voice
+ || source == MisalignmentTurnSource::AcknowledgedContinuation)
+ })
{
self.misalignment_policy_violation = None;
self.bottom_pane.dismiss_view_by_id(PRECAUTION_VIEW);
diff --git a/codex-rs/tui/src/chatwidget/protocol.rs b/codex-rs/tui/src/chatwidget/protocol.rs
index c2fc8039ef..d43adf4f53 100644
--- a/codex-rs/tui/src/chatwidget/protocol.rs
+++ b/codex-rs/tui/src/chatwidget/protocol.rs
@@ -61,7 +61,10 @@ impl ChatWidget {
}
ServerNotification::TurnStarted(notification) => {
if replay_kind.is_none() {
- self.clear_misalignment_for_new_turn(&notification.turn.id);
+ self.clear_misalignment_for_new_turn(
+ &notification.turn.id,
+ MisalignmentTurnSource::ServerNotification,
+ );
}
self.turn_lifecycle.last_turn_id = Some(notification.turn.id);
self.last_non_retry_error = None;
@@ -80,18 +83,43 @@ impl ChatWidget {
self.handle_item_completed_notification(notification, replay_kind);
}
ServerNotification::AgentMessageDelta(notification) => {
- self.on_agent_message_delta(notification.delta);
+ if !self.is_realtime_delegated_reasoning_turn(&notification.turn_id)
+ && (from_replay
+ || !self.is_realtime_delegated_agent_item(
+ &notification.turn_id,
+ &notification.item_id,
+ ))
+ {
+ self.on_agent_message_delta(notification.delta);
+ }
}
ServerNotification::PlanDelta(notification) => self.on_plan_delta(notification.delta),
ServerNotification::ReasoningSummaryTextDelta(notification) => {
- self.on_agent_reasoning_delta(notification.delta);
+ if !self.is_realtime_delegated_reasoning_item(
+ &notification.turn_id,
+ &notification.item_id,
+ ) {
+ self.on_agent_reasoning_delta(notification.delta);
+ }
}
ServerNotification::ReasoningTextDelta(notification) => {
- if self.config.show_raw_agent_reasoning {
+ if self.config.show_raw_agent_reasoning
+ && !self.is_realtime_delegated_reasoning_item(
+ &notification.turn_id,
+ &notification.item_id,
+ )
+ {
self.on_agent_reasoning_delta(notification.delta);
}
}
- ServerNotification::ReasoningSummaryPartAdded(_) => self.on_reasoning_section_break(),
+ ServerNotification::ReasoningSummaryPartAdded(notification) => {
+ if !self.is_realtime_delegated_reasoning_item(
+ &notification.turn_id,
+ &notification.item_id,
+ ) {
+ self.on_reasoning_section_break();
+ }
+ }
ServerNotification::TerminalInteraction(notification) => {
self.on_terminal_interaction(notification.process_id, notification.stdin)
}
@@ -235,6 +263,36 @@ impl ChatWidget {
self.on_shutdown_complete();
}
}
+ ServerNotification::ThreadRealtimeSdp(notification) => {
+ if !from_replay {
+ self.on_realtime_conversation_sdp(notification.sdp);
+ }
+ }
+ ServerNotification::ThreadRealtimeStarted(_) => {
+ if !from_replay {
+ self.on_realtime_conversation_started();
+ }
+ }
+ ServerNotification::ThreadRealtimeTranscriptDelta(notification) => {
+ if !from_replay {
+ self.on_realtime_transcript_delta(notification.role, notification.delta);
+ }
+ }
+ ServerNotification::ThreadRealtimeTranscriptDone(notification) => {
+ if !from_replay {
+ self.on_realtime_transcript_done(notification.role, notification.text);
+ }
+ }
+ ServerNotification::ThreadRealtimeError(notification) => {
+ if !from_replay {
+ self.on_realtime_error(notification.message);
+ }
+ }
+ ServerNotification::ThreadRealtimeClosed(notification) => {
+ if !from_replay {
+ self.on_realtime_conversation_closed(notification.reason);
+ }
+ }
ServerNotification::ServerRequestResolved(_)
| ServerNotification::AccountUpdated(_)
| ServerNotification::AccountRateLimitsUpdated(_)
@@ -264,17 +322,11 @@ impl ChatWidget {
| ServerNotification::TurnModerationMetadata(_)
| ServerNotification::FuzzyFileSearchSessionUpdated(_)
| ServerNotification::FuzzyFileSearchSessionCompleted(_)
- | ServerNotification::ThreadRealtimeStarted(_)
| ServerNotification::ThreadRealtimeItemAdded(_)
| ServerNotification::ThreadRealtimeItemStarted(_)
| ServerNotification::ThreadRealtimeItemTranscriptDelta(_)
| ServerNotification::ThreadRealtimeItemCompleted(_)
| ServerNotification::ThreadRealtimeOutputAudioDelta(_)
- | ServerNotification::ThreadRealtimeError(_)
- | ServerNotification::ThreadRealtimeClosed(_)
- | ServerNotification::ThreadRealtimeSdp(_)
- | ServerNotification::ThreadRealtimeTranscriptDelta(_)
- | ServerNotification::ThreadRealtimeTranscriptDone(_)
| ServerNotification::WindowsWorldWritableWarning(_)
| ServerNotification::WindowsSandboxSetupCompleted(_)
| ServerNotification::AccountLoginCompleted(_)
@@ -310,7 +362,12 @@ impl ChatWidget {
text,
phase: Some(MessagePhase::FinalAnswer) | None,
..
- } => Some((item.clone(), id.clone(), text.clone())),
+ } if !self
+ .is_realtime_delegated_reasoning_turn(&notification.turn.id)
+ || !realtime::is_private_realtime_agent_item(item) =>
+ {
+ Some((item.clone(), id.clone(), text.clone()))
+ }
_ => None,
});
if let Some((item, id, _)) = &last_agent_message
@@ -329,6 +386,11 @@ impl ChatWidget {
.map_or(ThreadItemRenderSource::Live, ThreadItemRenderSource::Replay),
);
}
+ if replay_kind.is_none()
+ && let Some((item, _, _)) = &last_agent_message
+ {
+ self.speak_completed_realtime_delegation(&notification.turn.id, item);
+ }
self.last_non_retry_error = None;
let completion = self.completion_cell(&notification.turn, replay_kind);
self.on_task_complete(
@@ -355,7 +417,10 @@ impl ChatWidget {
&& error.codex_error_info
== Some(AppServerCodexErrorInfo::MisalignmentPolicyViolation)
{
- self.on_misalignment_error(Some(notification.turn.id), error.misalignment);
+ self.on_misalignment_error(
+ Some(notification.turn.id.clone()),
+ error.misalignment,
+ );
self.last_non_retry_error = None;
} else if self.last_non_retry_error.as_ref()
== Some(&(notification.turn.id.clone(), error.message.clone()))
@@ -373,6 +438,9 @@ impl ChatWidget {
}
TurnStatus::InProgress => {}
}
+ if replay_kind.is_none() {
+ self.finish_realtime_turn(&notification.turn.id);
+ }
self.thread_usage.replaying_turn_completion = was_replaying_turn_completion;
}
@@ -382,6 +450,28 @@ impl ChatWidget {
replay_kind: Option<ReplayKind>,
) {
match notification.item {
+ ThreadItem::UserMessage { content, .. } if replay_kind.is_none() => {
+ self.note_realtime_user_item_started(&notification.turn_id, &content);
+ }
+ ThreadItem::UserMessage { content, .. }
+ if realtime::realtime_delegation_input(&content).is_some() =>
+ {
+ self.remember_realtime_delegated_reasoning_turn(&notification.turn_id);
+ }
+ ThreadItem::AgentMessage { id, .. } if replay_kind.is_none() => {
+ self.is_realtime_delegated_agent_item(&notification.turn_id, &id);
+ }
+ ThreadItem::Reasoning { id, .. }
+ if replay_kind.is_none()
+ && !self.is_realtime_delegated_reasoning_turn(&notification.turn_id) =>
+ {
+ // A later voice handoff can steer this turn without making an
+ // already-started typed reasoning item private.
+ self.realtime_conversation
+ .agent_items
+ .entry((notification.turn_id.clone(), id))
+ .or_insert(realtime::RealtimeAgentItemOrigin::Typed);
+ }
ThreadItem::ContextCompaction { id }
if !matches!(replay_kind, Some(ReplayKind::ResumeInitialMessages)) =>
{
@@ -441,6 +531,17 @@ impl ChatWidget {
notification: ItemCompletedNotification,
replay_kind: Option<ReplayKind>,
) {
+ if replay_kind.is_none()
+ && self.is_realtime_delegated_reasoning_turn(&notification.turn_id)
+ && realtime::is_private_realtime_agent_item(&notification.item)
+ && !matches!(&notification.item, ThreadItem::AgentMessage { id, .. } | ThreadItem::Reasoning { id, .. }
+ if matches!(
+ self.realtime_conversation.agent_items.get(&(notification.turn_id.clone(), id.clone())),
+ Some(realtime::RealtimeAgentItemOrigin::Typed)
+ ))
+ {
+ return;
+ }
// Buffered live notifications can introduce questions; historical turn replay cannot.
if replay_kind == Some(ReplayKind::ThreadSnapshot)
&& let ThreadItem::AgentMessage {
diff --git a/codex-rs/tui/src/chatwidget/realtime.rs b/codex-rs/tui/src/chatwidget/realtime.rs
new file mode 100644
index 0000000000..c37a4e2565
--- /dev/null
+++ b/codex-rs/tui/src/chatwidget/realtime.rs
@@ -0,0 +1,1785 @@
+//! TUI orchestration for an app-server-signaled, locally owned WebRTC voice session.
+//! Completed captions and both speakers' partials stay bounded across widget replacement.
+
+use super::ChatWidget;
+use super::HistoryCell;
+use super::PARENT_OWNED_INPUT_MESSAGE;
+use crate::app_command::AppCommand;
+use crate::app_event::AppEvent;
+use crate::history_cell;
+use codex_app_server_protocol::ThreadItem;
+use codex_app_server_protocol::UserInput;
+use codex_features::Feature;
+use codex_protocol::ThreadId;
+use codex_protocol::models::MessagePhase;
+use codex_realtime_webrtc::RealtimeWebrtcSession;
+use codex_realtime_webrtc::RealtimeWebrtcSessionHandle;
+use codex_realtime_webrtc::StartedRealtimeWebrtcSession;
+use futures::future::AbortHandle;
+use std::collections::HashMap;
+use std::collections::VecDeque;
+use std::collections::hash_map::DefaultHasher;
+use std::hash::Hash;
+use std::hash::Hasher;
+use std::sync::atomic::AtomicU64;
+use std::sync::atomic::Ordering;
+use std::time::Duration;
+use std::time::Instant;
+
+#[cfg(test)]
+#[path = "realtime_tests.rs"]
+pub(crate) mod tests;
+
+pub(crate) const MAX_TRANSCRIPT_BYTES: usize = 1024;
+const MAX_PENDING_TRANSCRIPT_CELLS: usize = 32;
+pub(crate) const MAX_REPLAY_TRANSCRIPT_CELLS: usize = MAX_PENDING_TRANSCRIPT_CELLS + 2;
+const MAX_PENDING_SPEECH_DELIVERIES: usize = 16;
+const MAX_RETAINED_DELEGATED_REASONING_TURNS: usize = 16;
+const MAX_PENDING_SPEECH_ITEM_BYTES: usize = 64 * 1024;
+const MAX_PENDING_SPEECH_TURN_ID_BYTES: usize = 512;
+// Leave room for Core's optional backend prefix within its 1,000-token speech limit.
+const MAX_SPEAKABLE_FINAL_TOKENS: usize = 990;
+const AUDIO_METER_SEGMENTS: usize = 5;
+const AUDIO_METER_NOISE_FLOOR: u16 = 512;
+const AUDIO_METER_FULL_SCALE: u16 = 8192;
+const MICROPHONE_METER_INTERVAL: Duration = Duration::from_millis(100);
+const SPEAKER_ACTIVITY_HOLD: Duration = Duration::from_millis(500);
+static NEXT_REALTIME_ATTEMPT_ID: AtomicU64 = AtomicU64::new(1);
+static NEXT_REALTIME_SPEECH_DELIVERY_ID: AtomicU64 = AtomicU64::new(1);
+
+struct PendingRealtimeSpeech {
+ state: PendingSpeechState,
+ captioned: bool,
+ input_generation: u64,
+ thread_id: ThreadId,
+ turn_id: String,
+ item: ThreadItem,
+}
+
+pub(crate) struct RealtimeTranscriptRecord {
+ pub(crate) role: String,
+ pub(crate) text: String,
+ pub(crate) complete: bool,
+}
+
+#[derive(Clone, Copy, PartialEq, Eq)]
+enum PendingSpeechState {
+ AwaitingTurn,
+ Queued(u64),
+ Accepted,
+}
+
+fn can_retain_realtime_speech(turn_id: &str, item: &ThreadItem) -> bool {
+ turn_id.len() <= MAX_PENDING_SPEECH_TURN_ID_BYTES
+ && serde_json::to_vec(item)
+ .is_ok_and(|encoded| encoded.len() <= MAX_PENDING_SPEECH_ITEM_BYTES)
+}
+
+#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
+enum RealtimeConversationPhase {
+ #[default]
+ Inactive,
+ Starting,
+ Active,
+ Stopping,
+}
+
+#[derive(Clone, Copy, Debug)]
+pub(super) enum RealtimeAgentItemOrigin {
+ Typed,
+ Delegated {
+ may_speak: bool,
+ completed: bool,
+ suppressed_nonfinal: bool,
+ input_generation: u64,
+ },
+}
+
+#[derive(Clone, Copy, Debug)]
+enum RealtimeTurnOrigin {
+ Typed {
+ input_generation: u64,
+ },
+ Delegated {
+ may_speak: bool,
+ input_generation: u64,
+ },
+}
+
+#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
+enum StartupRetry {
+ #[default]
+ Available,
+ WaitingForStop,
+ Used,
+}
+
+#[derive(Default)]
+pub(super) struct RealtimeConversationUiState {
+ startup_retry: StartupRetry,
+ phase: RealtimeConversationPhase,
+ recover_late_transcripts: bool,
+ attempt_id: u64,
+ thread_id: Option<ThreadId>,
+ pub(super) handle: Option<RealtimeWebrtcSessionHandle>,
+ startup_abort: Option<AbortHandle>,
+ backend_started: bool,
+ webrtc_connected: bool,
+ microphone_muted: bool,
+ microphone_level: usize,
+ speaker_level: usize,
+ speaker_active_until: Option<Instant>,
+ speaker_suppression_generation: Option<u64>,
+ transcript_role: Option<String>,
+ transcript: String,
+ // The other speaker's bounded partial while duplex deltas interleave.
+ interleaved_transcript: Option<(String, String)>,
+ transcript_input_generation: Option<u64>,
+ assistant_transcript_generation: Option<u64>,
+ assistant_caption_started_after_speech_queue: bool,
+ pub(super) live_transcript_cell: Option<Box<dyn HistoryCell>>,
+ pending_history_cells: VecDeque<Box<dyn HistoryCell>>,
+ accepted_transcripts: VecDeque<RealtimeTranscriptRecord>,
+ latest_input_was_voice: bool,
+ input_generation: u64,
+ latest_voice_input_fingerprint: Option<(usize, u64)>,
+ pending_typed_input: Option<String>,
+ turn_origins: HashMap<String, RealtimeTurnOrigin>,
+ delegated_reasoning_turns: VecDeque<String>,
+ pub(super) agent_items: HashMap<(String, String), RealtimeAgentItemOrigin>,
+ pending_speech: VecDeque<PendingRealtimeSpeech>,
+}
+
+pub(crate) fn realtime_delegation_input(items: &[UserInput]) -> Option<&str> {
+ let [
+ UserInput::Text {
+ text,
+ text_elements,
+ },
+ ] = items
+ else {
+ return None;
+ };
+ if !text_elements.is_empty() {
+ return None;
+ }
+ let body = text
+ .trim()
+ .strip_prefix("<realtime_delegation>")
+ .and_then(|body| body.strip_suffix("</realtime_delegation>"))?;
+ let (_, input) = body.split_once("<input>")?;
+ input.split_once("</input>").map(|(input, _)| input)
+}
+
+pub(crate) fn is_private_realtime_agent_item(item: &ThreadItem) -> bool {
+ if matches!(item, ThreadItem::Reasoning { .. }) {
+ return true;
+ }
+ let ThreadItem::AgentMessage { text, phase, .. } = item else {
+ return false;
+ };
+ let text = text.trim_start();
+ matches!(phase, Some(MessagePhase::Commentary))
+ || (!matches!(phase, Some(MessagePhase::FinalAnswer))
+ && (text.starts_with("[ANALYSIS]") || text.starts_with("[COMMENTARY]")))
+}
+
+pub(crate) fn realtime_delegation_display_text(input: &str) -> String {
+ input
+ .replace("&lt;", "<")
+ .replace("&gt;", ">")
+ .replace("&amp;", "&")
+}
+
+fn realtime_input_fingerprint(input: &str) -> (usize, u64) {
+ let input = input.trim();
+ let mut hasher = DefaultHasher::new();
+ input.hash(&mut hasher);
+ (input.len(), hasher.finish())
+}
+
+impl ChatWidget {
+ pub(crate) fn realtime_conversation_is_running(&self) -> bool {
+ self.realtime_conversation.phase != RealtimeConversationPhase::Inactive
+ }
+
+ pub(crate) fn may_receive_realtime_transcripts(&self) -> bool {
+ self.realtime_conversation.phase != RealtimeConversationPhase::Inactive
+ || self.realtime_conversation.recover_late_transcripts
+ }
+
+ pub(crate) fn toggle_realtime_conversation(&mut self) {
+ if self.realtime_conversation.phase == RealtimeConversationPhase::Stopping {
+ self.realtime_conversation.startup_retry = StartupRetry::Used;
+ self.add_info_message(
+ "Voice conversation is still stopping.".to_string(),
+ /*hint*/ None,
+ );
+ return;
+ }
+
+ if self.realtime_conversation.phase != RealtimeConversationPhase::Inactive {
+ self.stop_realtime_conversation();
+ return;
+ }
+
+ if self.has_misalignment_policy_violation() {
+ self.show_misalignment_policy_precaution();
+ return;
+ }
+
+ if self.side_conversation_active() {
+ self.add_error_message(
+ "Voice mode is unavailable in side conversations. Return to the main thread first."
+ .to_string(),
+ );
+ return;
+ }
+
+ if self.blocks_direct_input {
+ self.add_error_message(PARENT_OWNED_INPUT_MESSAGE.to_string());
+ return;
+ }
+
+ if !self.config.features.enabled(Feature::RealtimeConversation) {
+ self.add_error_message("Voice conversations are not enabled.".to_string());
+ return;
+ }
+
+ if !RealtimeWebrtcSession::is_supported() {
+ self.add_error_message(
+ "Voice requires macOS, an MSVC-based Windows build, or a glibc-based Linux build."
+ .to_string(),
+ );
+ return;
+ }
+
+ let Some(thread_id) = self.thread_id() else {
+ self.add_error_message("Start a conversation before using voice mode.".to_string());
+ return;
+ };
+
+ self.start_realtime_conversation(thread_id);
+ }
+
+ fn start_realtime_conversation(&mut self, thread_id: ThreadId) {
+ self.realtime_conversation.recover_late_transcripts = false;
+ self.realtime_conversation.attempt_id =
+ NEXT_REALTIME_ATTEMPT_ID.fetch_add(1, Ordering::Relaxed);
+ self.realtime_conversation.thread_id = Some(thread_id);
+ let attempt_id = self.realtime_conversation.attempt_id;
+ let (startup_abort, abort_registration) = AbortHandle::new_pair();
+ self.realtime_conversation.startup_abort = Some(startup_abort);
+ self.realtime_conversation.phase = RealtimeConversationPhase::Starting;
+ self.update_realtime_footer();
+ let app_event_tx = self.app_event_tx.clone();
+ std::thread::spawn(move || {
+ let result =
+ RealtimeWebrtcSession::start(abort_registration).map_err(|error| error.to_string());
+ app_event_tx.send(AppEvent::RealtimeWebrtcOfferCreated {
+ thread_id,
+ attempt_id,
+ result,
+ });
+ });
+ self.request_redraw();
+ }
+
+ pub(super) fn stop_realtime_conversation(&mut self) {
+ self.realtime_conversation.startup_retry = StartupRetry::Used;
+ if matches!(
+ self.realtime_conversation.phase,
+ RealtimeConversationPhase::Inactive | RealtimeConversationPhase::Stopping
+ ) {
+ return;
+ }
+ self.restore_all_undelivered_realtime_speech();
+
+ // Stopping may prevent final transcript events from arriving.
+ self.finish_realtime_partial_transcripts();
+
+ if let Some(handle) = self.realtime_conversation.handle.take() {
+ handle.close();
+ self.realtime_conversation.phase = RealtimeConversationPhase::Stopping;
+ self.set_footer_hint_override(/*items*/ None);
+ let Some(thread_id) = self.realtime_conversation.thread_id else {
+ self.reset_realtime_conversation();
+ return;
+ };
+ if !self.submit_op(AppCommand::RealtimeConversationStop { thread_id }) {
+ self.reset_realtime_conversation();
+ }
+ } else {
+ if let Some(thread_id) = self.reset_realtime_conversation() {
+ self.submit_op(AppCommand::RealtimeConversationStop { thread_id });
+ }
+ }
+ self.request_redraw();
+ }
+
+ pub(super) fn toggle_realtime_microphone(&mut self) {
+ let muted = !self.realtime_conversation.microphone_muted;
+ if let Some(handle) = self.realtime_conversation.handle.as_ref() {
+ if let Err(error) = handle.set_microphone_muted(muted) {
+ self.on_realtime_error(format!("Failed to update microphone: {error}"));
+ return;
+ }
+ } else if self.realtime_conversation.phase != RealtimeConversationPhase::Starting {
+ self.add_error_message("Start voice mode before muting the microphone.".to_string());
+ return;
+ }
+
+ self.realtime_conversation.microphone_muted = muted;
+ self.realtime_conversation.microphone_level = 0;
+ self.update_realtime_footer();
+ }
+
+ pub(crate) fn on_realtime_webrtc_offer_created(
+ &mut self,
+ thread_id: ThreadId,
+ attempt_id: u64,
+ result: Result<StartedRealtimeWebrtcSession, String>,
+ ) {
+ if self.realtime_conversation.phase != RealtimeConversationPhase::Starting
+ || self.realtime_conversation.attempt_id != attempt_id
+ {
+ if let Ok(offer) = result {
+ offer.handle.close();
+ }
+ return;
+ }
+
+ let offer = match result {
+ Ok(offer) => offer,
+ Err(error) => {
+ self.on_realtime_error(format!("Failed to start voice mode: {error}"));
+ return;
+ }
+ };
+ self.realtime_conversation.startup_abort = None;
+ if self.realtime_conversation.microphone_muted
+ && let Err(error) = offer.handle.set_microphone_muted(/*muted*/ true)
+ {
+ offer.handle.close();
+ self.on_realtime_error(format!("Failed to restore microphone mute: {error}"));
+ return;
+ }
+ self.realtime_conversation.handle = Some(offer.handle);
+ self.frame_requester
+ .schedule_frame_in(MICROPHONE_METER_INTERVAL);
+ if !self.submit_op(AppCommand::RealtimeConversationStart {
+ thread_id,
+ offer_sdp: offer.offer_sdp.into(),
+ }) {
+ self.reset_realtime_conversation();
+ }
+ }
+
+ pub(super) fn on_realtime_conversation_sdp(&mut self, answer_sdp: String) {
+ if self.realtime_conversation.phase != RealtimeConversationPhase::Starting {
+ return;
+ }
+ let Some(handle) = self.realtime_conversation.handle.clone() else {
+ return;
+ };
+ let Some(thread_id) = self.thread_id() else {
+ return;
+ };
+ let attempt_id = self.realtime_conversation.attempt_id;
+ let app_event_tx = self.app_event_tx.clone();
+ std::thread::spawn(move || {
+ let result = handle.apply_answer_sdp(answer_sdp);
+ app_event_tx.send(AppEvent::RealtimeWebrtcConnected {
+ thread_id,
+ attempt_id,
+ result,
+ });
+ });
+ }
+
+ pub(super) fn on_realtime_conversation_started(&mut self) {
+ if self.realtime_conversation.phase != RealtimeConversationPhase::Starting {
+ return;
+ }
+ self.realtime_conversation.backend_started = true;
+ self.maybe_activate_realtime_conversation();
+ }
+
+ pub(crate) fn on_realtime_webrtc_connected(
+ &mut self,
+ attempt_id: u64,
+ result: Result<(), codex_realtime_webrtc::ConnectionError>,
+ ) {
+ if self.realtime_conversation.phase != RealtimeConversationPhase::Starting
+ || self.realtime_conversation.attempt_id != attempt_id
+ {
+ return;
+ }
+ if let Err(error) = result {
+ if error == codex_realtime_webrtc::ConnectionError::NegotiationTimedOut
+ && self.realtime_conversation.startup_retry == StartupRetry::Available
+ && let Some(thread_id) = self.realtime_conversation.thread_id
+ {
+ if let Some(handle) = self.realtime_conversation.handle.take() {
+ handle.close();
+ }
+ self.realtime_conversation.phase = RealtimeConversationPhase::Stopping;
+ self.realtime_conversation.startup_retry = StartupRetry::WaitingForStop;
+ if !self.submit_op(AppCommand::RealtimeConversationStop { thread_id }) {
+ self.reset_realtime_conversation();
+ } else {
+ self.add_info_message(
+ "Voice connection timed out. Retrying once after cleanup.".into(),
+ /*hint*/ None,
+ );
+ }
+ return;
+ }
+ self.on_realtime_error(format!("Failed to connect voice mode: {error}"));
+ return;
+ }
+
+ self.realtime_conversation.webrtc_connected = true;
+ self.maybe_activate_realtime_conversation();
+ }
+
+ fn maybe_activate_realtime_conversation(&mut self) {
+ if self.realtime_conversation.phase != RealtimeConversationPhase::Starting
+ || !self.realtime_conversation.backend_started
+ || !self.realtime_conversation.webrtc_connected
+ {
+ return;
+ }
+ self.realtime_conversation.phase = RealtimeConversationPhase::Active;
+ let running_delegation =
+ self.turn_lifecycle
+ .last_turn_id
+ .as_deref()
+ .is_some_and(|turn_id| {
+ matches!(
+ self.realtime_conversation.turn_origins.get(turn_id),
+ Some(RealtimeTurnOrigin::Delegated {
+ may_speak: true,
+ ..
+ })
+ )
+ });
+ self.realtime_conversation.latest_input_was_voice =
+ !self.turn_lifecycle.agent_turn_running || running_delegation;
+ if self.turn_lifecycle.agent_turn_running
+ && let Some(turn_id) = self.turn_lifecycle.last_turn_id.clone()
+ {
+ self.realtime_conversation
+ .turn_origins
+ .entry(turn_id)
+ .or_insert(RealtimeTurnOrigin::Typed {
+ input_generation: self.realtime_conversation.input_generation,
+ });
+ }
+ self.update_realtime_footer();
+ self.frame_requester
+ .schedule_frame_in(MICROPHONE_METER_INTERVAL);
+ self.add_info_message(
+ "Voice conversation started.".to_string(),
+ Some("Use /voice mute to mute or /voice to stop.".to_string()),
+ );
+ }
+
+ pub(crate) fn is_current_realtime_attempt(
+ &self,
+ thread_id: ThreadId,
+ attempt_id: u64,
+ input_generation: u64,
+ ) -> bool {
+ self.realtime_conversation.phase == RealtimeConversationPhase::Active
+ && self.realtime_conversation.thread_id == Some(thread_id)
+ && self.realtime_conversation.attempt_id == attempt_id
+ && self.realtime_conversation.input_generation == input_generation
+ && self.realtime_conversation.latest_input_was_voice
+ }
+
+ pub(super) fn note_realtime_typed_input(&mut self, text: &str) {
+ self.invalidate_realtime_voice_input();
+ if self.realtime_conversation.phase == RealtimeConversationPhase::Active {
+ self.realtime_conversation.pending_typed_input = Some(text.to_string());
+ }
+ }
+
+ fn invalidate_realtime_voice_input(&mut self) {
+ if self.realtime_conversation.phase != RealtimeConversationPhase::Active {
+ return;
+ }
+ self.realtime_conversation.latest_input_was_voice = false;
+ self.realtime_conversation.input_generation = self
+ .realtime_conversation
+ .input_generation
+ .wrapping_add(/*rhs*/ 1);
+ // Typing supersedes the old voice answer. Suppress the helper before
+ // another queued audio frame can play, even when the mic is muted.
+ self.realtime_conversation.speaker_suppression_generation =
+ Some(self.realtime_conversation.input_generation);
+ if let Some(handle) = self.realtime_conversation.handle.as_ref() {
+ handle.set_speaker_suppressed(/*suppressed*/ true);
+ }
+ self.realtime_conversation.speaker_level = 0;
+ self.realtime_conversation.speaker_active_until = None;
+ self.update_realtime_footer();
+ for origin in self.realtime_conversation.turn_origins.values_mut() {
+ if let RealtimeTurnOrigin::Delegated { may_speak, .. } = origin {
+ *may_speak = false;
+ }
+ }
+ for origin in self.realtime_conversation.agent_items.values_mut() {
+ if let RealtimeAgentItemOrigin::Delegated { may_speak, .. } = origin {
+ *may_speak = false;
+ }
+ }
+ }
+
+ fn release_realtime_speaker(&mut self) {
+ self.realtime_conversation.speaker_suppression_generation = None;
+ if let Some(handle) = self.realtime_conversation.handle.as_ref() {
+ handle.set_speaker_suppressed(/*suppressed*/ false);
+ }
+ }
+
+ fn suppress_realtime_speaker(&mut self) {
+ self.realtime_conversation.speaker_suppression_generation =
+ Some(self.realtime_conversation.input_generation);
+ if let Some(handle) = self.realtime_conversation.handle.as_ref() {
+ handle.set_speaker_suppressed(/*suppressed*/ true);
+ }
+ self.realtime_conversation.speaker_level = 0;
+ self.realtime_conversation.speaker_active_until = None;
+ self.update_realtime_footer();
+ }
+
+ fn resume_realtime_speaker_for(&mut self, role: &str, text: &str) {
+ if role == "assistant"
+ && !text.trim().is_empty()
+ && self.realtime_conversation.assistant_transcript_generation
+ == Some(self.realtime_conversation.input_generation)
+ && self.realtime_conversation.latest_input_was_voice
+ && self.realtime_conversation.speaker_suppression_generation
+ == Some(self.realtime_conversation.input_generation)
+ {
+ self.release_realtime_speaker();
+ }
+ }
+
+ pub(super) fn note_realtime_user_item_started(&mut self, turn_id: &str, items: &[UserInput]) {
+ if !matches!(
+ self.realtime_conversation.phase,
+ RealtimeConversationPhase::Starting | RealtimeConversationPhase::Active
+ ) {
+ if realtime_delegation_input(items).is_some() {
+ self.remember_realtime_delegated_reasoning_turn(turn_id);
+ }
+ return;
+ }
+ let matches_local_typed_input = self
+ .realtime_conversation
+ .pending_typed_input
+ .as_ref()
+ .is_some_and(|pending| {
+ items
+ .iter()
+ .any(|item| matches!(item, UserInput::Text { text, .. } if text == pending))
+ });
+ if matches_local_typed_input {
+ self.realtime_conversation.pending_typed_input = None;
+ }
+ if let Some(input) = realtime_delegation_input(items)
+ && !matches_local_typed_input
+ {
+ let transcript_tail_flush = items.iter().any(|item| {
+ matches!(item, UserInput::Text { text, .. }
+ if text.contains("<source>transcript_tail_flush</source>"))
+ });
+ let stale_voice_input = !self.realtime_conversation.latest_input_was_voice
+ && self
+ .realtime_conversation
+ .latest_voice_input_fingerprint
+ .is_some_and(|latest| {
+ latest
+ == realtime_input_fingerprint(&realtime_delegation_display_text(input))
+ });
+ let (typed_turn_generation, delegated_turn_generation) =
+ match self.realtime_conversation.turn_origins.get(turn_id) {
+ Some(RealtimeTurnOrigin::Typed { input_generation }) => {
+ (Some(*input_generation), None)
+ }
+ Some(RealtimeTurnOrigin::Delegated {
+ input_generation, ..
+ }) => (None, Some(*input_generation)),
+ None => (None, None),
+ };
+ let voice_supersedes_typed_turn = typed_turn_generation.is_some_and(|generation| {
+ generation != self.realtime_conversation.input_generation
+ || !self.realtime_conversation.latest_input_was_voice
+ }) && !transcript_tail_flush
+ && !stale_voice_input;
+ let typed_turn = typed_turn_generation.is_some() && !voice_supersedes_typed_turn;
+ let may_speak = !transcript_tail_flush && !stale_voice_input && !typed_turn;
+ if may_speak {
+ let replaces_current_delegation = self
+ .realtime_conversation
+ .turn_origins
+ .values()
+ .any(|origin| {
+ matches!(
+ origin,
+ RealtimeTurnOrigin::Delegated {
+ may_speak: true,
+ input_generation,
+ } if *input_generation == self.realtime_conversation.input_generation
+ )
+ });
+ if !self.realtime_conversation.latest_input_was_voice || replaces_current_delegation
+ {
+ self.realtime_conversation.input_generation = self
+ .realtime_conversation
+ .input_generation
+ .wrapping_add(/*rhs*/ 1);
+ if self
+ .realtime_conversation
+ .speaker_suppression_generation
+ .is_some()
+ {
+ self.realtime_conversation.speaker_suppression_generation =
+ Some(self.realtime_conversation.input_generation);
+ }
+ }
+ self.realtime_conversation.latest_input_was_voice = true;
+ }
+ let delegated_origin = RealtimeTurnOrigin::Delegated {
+ may_speak,
+ input_generation: self.realtime_conversation.input_generation,
+ };
+ let supersedes_previous_delegation = may_speak
+ && delegated_turn_generation.is_some_and(|generation| {
+ generation != self.realtime_conversation.input_generation
+ });
+ if voice_supersedes_typed_turn || supersedes_previous_delegation {
+ self.realtime_conversation
+ .turn_origins
+ .insert(turn_id.to_string(), delegated_origin);
+ } else {
+ self.realtime_conversation
+ .turn_origins
+ .entry(turn_id.to_string())
+ .or_insert(delegated_origin);
+ }
+ if matches!(
+ self.realtime_conversation.turn_origins.get(turn_id),
+ Some(RealtimeTurnOrigin::Delegated { .. })
+ ) {
+ self.remember_realtime_delegated_reasoning_turn(turn_id);
+ }
+ return;
+ }
+
+ self.invalidate_realtime_voice_input();
+ self.realtime_conversation.turn_origins.insert(
+ turn_id.to_string(),
+ RealtimeTurnOrigin::Typed {
+ input_generation: self.realtime_conversation.input_generation,
+ },
+ );
+ self.realtime_conversation
+ .delegated_reasoning_turns
+ .retain(|saved| saved != turn_id);
+ }
+
+ pub(crate) fn remember_realtime_delegated_reasoning_turn(&mut self, turn_id: &str) {
+ if turn_id.len() > 512
+ || self
+ .realtime_conversation
+ .delegated_reasoning_turns
+ .iter()
+ .any(|saved| saved == turn_id)
+ {
+ return;
+ }
+ self.realtime_conversation
+ .delegated_reasoning_turns
+ .push_back(turn_id.to_string());
+ if self.realtime_conversation.delegated_reasoning_turns.len()
+ > MAX_RETAINED_DELEGATED_REASONING_TURNS
+ {
+ self.realtime_conversation
+ .delegated_reasoning_turns
+ .pop_front();
+ }
+ }
+
+ pub(super) fn is_realtime_delegated_reasoning_turn(&self, turn_id: &str) -> bool {
+ self.realtime_conversation
+ .delegated_reasoning_turns
+ .iter()
+ .any(|saved| saved == turn_id)
+ }
+
+ pub(super) fn is_realtime_delegated_reasoning_item(
+ &self,
+ turn_id: &str,
+ item_id: &str,
+ ) -> bool {
+ self.is_realtime_delegated_reasoning_turn(turn_id)
+ && !matches!(
+ self.realtime_conversation
+ .agent_items
+ .get(&(turn_id.to_string(), item_id.to_string())),
+ Some(RealtimeAgentItemOrigin::Typed)
+ )
+ }
+
+ pub(super) fn should_hide_realtime_delegation(&self, turn_id: &str) -> bool {
+ matches!(
+ self.realtime_conversation.phase,
+ RealtimeConversationPhase::Starting | RealtimeConversationPhase::Active
+ ) && matches!(
+ self.realtime_conversation.turn_origins.get(turn_id),
+ Some(RealtimeTurnOrigin::Delegated { .. })
+ )
+ }
+
+ pub(super) fn is_realtime_delegated_agent_item(
+ &mut self,
+ turn_id: &str,
+ item_id: &str,
+ ) -> bool {
+ if !matches!(
+ self.realtime_conversation.phase,
+ RealtimeConversationPhase::Starting | RealtimeConversationPhase::Active
+ ) {
+ return false;
+ }
+ let default_origin = match self.realtime_conversation.turn_origins.get(turn_id) {
+ Some(RealtimeTurnOrigin::Delegated {
+ may_speak,
+ input_generation,
+ }) => RealtimeAgentItemOrigin::Delegated {
+ may_speak: *may_speak,
+ completed: false,
+ suppressed_nonfinal: false,
+ input_generation: *input_generation,
+ },
+ Some(RealtimeTurnOrigin::Typed { .. }) | None => RealtimeAgentItemOrigin::Typed,
+ };
+ matches!(
+ self.realtime_conversation
+ .agent_items
+ .entry((turn_id.to_string(), item_id.to_string()))
+ .or_insert(default_origin),
+ RealtimeAgentItemOrigin::Delegated { .. }
+ )
+ }
+
+ pub(super) fn complete_realtime_delegated_agent_item(
+ &mut self,
+ turn_id: &str,
+ item: &ThreadItem,
+ from_replay: bool,
+ ) -> bool {
+ let ThreadItem::AgentMessage {
+ id: item_id,
+ text,
+ phase,
+ questions,
+ ..
+ } = item
+ else {
+ return false;
+ };
+ // The ordinary completion path opens the structured question editor.
+ // A question-bearing answer must remain visible instead of being speech-only.
+ if questions.is_some() {
+ return false;
+ }
+ if from_replay || !self.is_realtime_delegated_agent_item(turn_id, item_id) {
+ return false;
+ }
+ let Some(RealtimeAgentItemOrigin::Delegated {
+ may_speak,
+ completed,
+ suppressed_nonfinal,
+ ..
+ }) = self
+ .realtime_conversation
+ .agent_items
+ .get_mut(&(turn_id.to_string(), item_id.to_string()))
+ else {
+ return false;
+ };
+ if *completed {
+ return true;
+ }
+ // The delegated agent's commentary and reasoning are private to the
+ // voice handoff. Never put them in the fallback queue, even when the
+ // item is too large to retain.
+ let trimmed = text.trim();
+ let explicitly_final = matches!(phase, Some(MessagePhase::FinalAnswer));
+ if is_private_realtime_agent_item(item)
+ || (!explicitly_final && trimmed == "[FINAL]")
+ || trimmed.is_empty()
+ {
+ *completed = true;
+ *suppressed_nonfinal = true;
+ return true;
+ }
+ // Typing ends the old item's speech ownership, but it does not make
+ // private voice commentary into typed output. Only a visible answer
+ // returns to the ordinary completion path.
+ if !*may_speak {
+ return false;
+ }
+ // A final item that cannot be retained for delivery recovery stays in
+ // ordinary history instead of becoming a voice-only answer.
+ if !can_retain_realtime_speech(turn_id, item) {
+ return false;
+ }
+ let Some(thread_id) = self.realtime_conversation.thread_id else {
+ return false;
+ };
+ let input_generation = self.realtime_conversation.input_generation;
+ *completed = true;
+ self.transcript.last_completed_agent_message =
+ Some((turn_id.to_string(), item_id.to_string()));
+ if self.realtime_conversation.pending_speech.len() >= MAX_PENDING_SPEECH_DELIVERIES
+ && let Some(oldest) = self.realtime_conversation.pending_speech.pop_front()
+ {
+ self.restore_realtime_speech(oldest);
+ }
+ self.realtime_conversation
+ .pending_speech
+ .push_back(PendingRealtimeSpeech {
+ state: PendingSpeechState::AwaitingTurn,
+ captioned: false,
+ input_generation,
+ thread_id,
+ turn_id: turn_id.to_string(),
+ item: item.clone(),
+ });
+
+ true
+ }
+
+ pub(super) fn speak_completed_realtime_delegation(&mut self, turn_id: &str, item: &ThreadItem) {
+ let ThreadItem::AgentMessage {
+ id: item_id, text, ..
+ } = item
+ else {
+ return;
+ };
+ let Some(RealtimeAgentItemOrigin::Delegated {
+ may_speak: true,
+ completed: true,
+ suppressed_nonfinal,
+ input_generation,
+ }) = self
+ .realtime_conversation
+ .agent_items
+ .get(&(turn_id.to_string(), item_id.to_string()))
+ else {
+ return;
+ };
+ let input_generation = *input_generation;
+ let suppressed_nonfinal = *suppressed_nonfinal;
+ if input_generation != self.realtime_conversation.input_generation {
+ return;
+ }
+ let trimmed = text.trim();
+ if is_private_realtime_agent_item(item)
+ || trimmed.is_empty()
+ || !self.realtime_conversation.latest_input_was_voice
+ {
+ return;
+ }
+ let Some(thread_id) = self.realtime_conversation.thread_id else {
+ return;
+ };
+ if self.thread_id() != Some(thread_id) {
+ return;
+ }
+ let text = trimmed
+ .strip_prefix("[FINAL]")
+ .map(str::trim_start)
+ .unwrap_or(trimmed)
+ .to_string();
+ if text.is_empty() {
+ return;
+ }
+ let Some(RealtimeTurnOrigin::Delegated {
+ may_speak,
+ input_generation: turn_input_generation,
+ }) = self.realtime_conversation.turn_origins.get_mut(turn_id)
+ else {
+ return;
+ };
+ if !*may_speak || *turn_input_generation != input_generation {
+ return;
+ }
+ *may_speak = false;
+ if !can_retain_realtime_speech(turn_id, item)
+ || codex_utils_string::approx_token_count(&text) > MAX_SPEAKABLE_FINAL_TOKENS
+ {
+ self.remove_waiting_realtime_speech(turn_id, item_id);
+ self.finish_realtime_turn(turn_id);
+ self.handle_thread_item(
+ item.clone(),
+ turn_id.to_string(),
+ super::ThreadItemRenderSource::Live,
+ );
+ return;
+ }
+ let delivery_id = NEXT_REALTIME_SPEECH_DELIVERY_ID.fetch_add(1, Ordering::Relaxed);
+ let index = self
+ .realtime_conversation
+ .pending_speech
+ .iter_mut()
+ .position(|pending| {
+ pending.turn_id == turn_id
+ && matches!(&pending.item, ThreadItem::AgentMessage { id, .. } if id == item_id)
+ && pending.state == PendingSpeechState::AwaitingTurn
+ });
+ let index = match index {
+ Some(index) => index,
+ None if suppressed_nonfinal => {
+ if self.realtime_conversation.pending_speech.len() >= MAX_PENDING_SPEECH_DELIVERIES
+ && let Some(oldest) = self.realtime_conversation.pending_speech.pop_front()
+ {
+ self.restore_realtime_speech(oldest);
+ }
+ self.realtime_conversation
+ .pending_speech
+ .push_back(PendingRealtimeSpeech {
+ state: PendingSpeechState::AwaitingTurn,
+ captioned: false,
+ input_generation,
+ thread_id,
+ turn_id: turn_id.to_string(),
+ item: item.clone(),
+ });
+ self.realtime_conversation.pending_speech.len() - 1
+ }
+ None => return,
+ };
+ let pending = &mut self.realtime_conversation.pending_speech[index];
+ pending.item = item.clone();
+ pending.state = PendingSpeechState::Queued(delivery_id);
+ if !self.submit_op(AppCommand::RealtimeConversationSpeech {
+ thread_id,
+ attempt_id: self.realtime_conversation.attempt_id,
+ input_generation,
+ delivery_id,
+ text: text.into(),
+ }) {
+ self.restore_undelivered_realtime_speech(delivery_id);
+ self.on_realtime_error("Failed to deliver the voice response.".to_string());
+ } else {
+ self.realtime_conversation.pending_speech.retain(|pending| {
+ pending.turn_id != turn_id || pending.state != PendingSpeechState::AwaitingTurn
+ });
+ }
+ }
+
+ pub(crate) fn has_pending_realtime_speech(&self, delivery_id: u64) -> bool {
+ self.realtime_conversation
+ .pending_speech
+ .iter()
+ .any(|delivery| delivery.state == PendingSpeechState::Queued(delivery_id))
+ }
+
+ pub(crate) fn take_undelivered_realtime_speech_for_replay(
+ &mut self,
+ ) -> Vec<(ThreadId, String, ThreadItem)> {
+ self.realtime_conversation
+ .pending_speech
+ .drain(..)
+ .filter(|delivery| !delivery.captioned)
+ .map(|delivery| (delivery.thread_id, delivery.turn_id, delivery.item))
+ .collect()
+ }
+
+ pub(crate) fn take_realtime_transcript_cells_for_replay(
+ &mut self,
+ ) -> VecDeque<RealtimeTranscriptRecord> {
+ let mut records = std::mem::take(&mut self.realtime_conversation.accepted_transcripts);
+ // Accepted records include cells already flushed into the old widget as
+ // well as those still deferred behind another assistant stream.
+ self.realtime_conversation.pending_history_cells.clear();
+ let mut retain_partial = |role: String, text: String| {
+ if text.trim().is_empty() {
+ return;
+ }
+ if let Some(partial) = records
+ .iter_mut()
+ .rev()
+ .find(|record| record.role == role && !record.complete)
+ {
+ partial.text = text;
+ return;
+ }
+ if records.len() >= MAX_REPLAY_TRANSCRIPT_CELLS {
+ records.pop_front();
+ }
+ records.push_back(RealtimeTranscriptRecord {
+ role,
+ text,
+ complete: false,
+ });
+ };
+ if let Some((role, text)) = self.realtime_conversation.interleaved_transcript.take() {
+ retain_partial(role, text);
+ }
+ if let Some(role) = self.realtime_conversation.transcript_role.take() {
+ let text = std::mem::take(&mut self.realtime_conversation.transcript);
+ retain_partial(role, text);
+ self.realtime_conversation.live_transcript_cell = None;
+ }
+ records
+ }
+
+ pub(crate) fn restore_realtime_transcript_cells(
+ &mut self,
+ records: VecDeque<RealtimeTranscriptRecord>,
+ ) {
+ self.realtime_conversation.recover_late_transcripts = true;
+ for record in records {
+ if self.realtime_conversation.accepted_transcripts.len() >= MAX_REPLAY_TRANSCRIPT_CELLS
+ {
+ self.realtime_conversation.accepted_transcripts.pop_front();
+ }
+ if record.complete {
+ if self.realtime_conversation.pending_history_cells.len()
+ >= MAX_REPLAY_TRANSCRIPT_CELLS
+ {
+ self.realtime_conversation.pending_history_cells.pop_front();
+ }
+ let cell = self.realtime_transcript_history_cell(&record.role, &record.text);
+ self.realtime_conversation
+ .pending_history_cells
+ .push_back(cell);
+ } else {
+ // Keep an unfinished caption editable until its late completion or close.
+ self.on_realtime_transcript_delta(record.role.clone(), record.text.clone());
+ }
+ self.realtime_conversation
+ .accepted_transcripts
+ .push_back(record);
+ }
+ self.flush_realtime_transcript_history();
+ }
+
+ pub(crate) fn render_undelivered_realtime_speech(
+ &mut self,
+ thread_id: ThreadId,
+ turn_id: String,
+ item: ThreadItem,
+ ) {
+ if self.thread_id() == Some(thread_id) {
+ self.handle_thread_item(item, turn_id, super::ThreadItemRenderSource::Live);
+ }
+ }
+
+ pub(crate) fn accept_realtime_speech(&mut self, delivery_id: u64) {
+ if let Some(delivery) = self
+ .realtime_conversation
+ .pending_speech
+ .iter_mut()
+ .find(|delivery| delivery.state == PendingSpeechState::Queued(delivery_id))
+ {
+ delivery.state = PendingSpeechState::Accepted;
+ }
+ self.realtime_conversation
+ .pending_speech
+ .retain(|delivery| !delivery.captioned);
+ }
+
+ pub(crate) fn restore_undelivered_realtime_speech(&mut self, delivery_id: u64) {
+ if let Some(index) = self
+ .realtime_conversation
+ .pending_speech
+ .iter()
+ .position(|delivery| delivery.state == PendingSpeechState::Queued(delivery_id))
+ && let Some(delivery) = self.realtime_conversation.pending_speech.remove(index)
+ {
+ self.restore_realtime_speech(delivery);
+ }
+ }
+
+ fn restore_realtime_speech(&mut self, delivery: PendingRealtimeSpeech) {
+ if delivery.captioned {
+ return;
+ }
+ if self.thread_id() != Some(delivery.thread_id) {
+ return;
+ }
+ self.forget_realtime_turn_origin(&delivery.turn_id);
+ self.handle_thread_item(
+ delivery.item,
+ delivery.turn_id,
+ super::ThreadItemRenderSource::Live,
+ );
+ }
+
+ fn restore_all_undelivered_realtime_speech(&mut self) {
+ while let Some(delivery) = self.realtime_conversation.pending_speech.pop_front() {
+ self.restore_realtime_speech(delivery);
+ }
+ }
+
+ pub(super) fn finish_realtime_turn(&mut self, turn_id: &str) {
+ let mut waiting = Vec::new();
+ let mut index = 0;
+ while index < self.realtime_conversation.pending_speech.len() {
+ if self.realtime_conversation.pending_speech[index].turn_id == turn_id
+ && self.realtime_conversation.pending_speech[index].state
+ == PendingSpeechState::AwaitingTurn
+ {
+ if let Some(delivery) = self.realtime_conversation.pending_speech.remove(index) {
+ waiting.push(delivery);
+ }
+ } else {
+ index += 1;
+ }
+ }
+ self.forget_realtime_turn_origin(turn_id);
+ for delivery in waiting {
+ if self.thread_id() == Some(delivery.thread_id) {
+ self.handle_thread_item(
+ delivery.item,
+ delivery.turn_id,
+ super::ThreadItemRenderSource::Live,
+ );
+ }
+ }
+ }
+
+ fn remove_waiting_realtime_speech(&mut self, turn_id: &str, item_id: &str) {
+ self.realtime_conversation.pending_speech.retain(|pending| {
+ pending.turn_id != turn_id
+ || pending.state != PendingSpeechState::AwaitingTurn
+ || !matches!(&pending.item, ThreadItem::AgentMessage { id, .. } if id == item_id)
+ });
+ }
+
+ fn forget_realtime_turn_origin(&mut self, turn_id: &str) {
+ self.realtime_conversation.turn_origins.remove(turn_id);
+ self.realtime_conversation
+ .delegated_reasoning_turns
+ .retain(|saved| saved != turn_id);
+ self.realtime_conversation
+ .agent_items
+ .retain(|(item_turn_id, _), _| item_turn_id != turn_id);
+ }
+
+ pub(super) fn on_realtime_transcript_delta(&mut self, role: String, delta: String) {
+ if self.realtime_conversation.phase == RealtimeConversationPhase::Inactive
+ && !self.realtime_conversation.recover_late_transcripts
+ {
+ return;
+ }
+ let active = matches!(
+ self.realtime_conversation.phase,
+ RealtimeConversationPhase::Starting | RealtimeConversationPhase::Active
+ );
+ // Duplex transcript chunks can interleave. Display role changes are not input boundaries.
+ if active
+ && role == "user"
+ && self
+ .realtime_conversation
+ .transcript_input_generation
+ .is_none()
+ {
+ self.realtime_conversation.input_generation = self
+ .realtime_conversation
+ .input_generation
+ .wrapping_add(/*rhs*/ 1);
+ self.realtime_conversation.transcript_input_generation =
+ Some(self.realtime_conversation.input_generation);
+ self.suppress_realtime_speaker();
+ }
+ if active
+ && role == "assistant"
+ && !delta.trim().is_empty()
+ && self
+ .realtime_conversation
+ .assistant_transcript_generation
+ .is_none()
+ {
+ self.realtime_conversation.assistant_transcript_generation =
+ Some(self.realtime_conversation.input_generation);
+ self.realtime_conversation
+ .assistant_caption_started_after_speech_queue = self
+ .realtime_conversation
+ .pending_speech
+ .iter()
+ .any(|delivery| {
+ delivery.input_generation == self.realtime_conversation.input_generation
+ && delivery.state != PendingSpeechState::AwaitingTurn
+ });
+ }
+ if active {
+ self.resume_realtime_speaker_for(&role, &delta);
+ }
+ if active
+ && role == "user"
+ && self.realtime_conversation.transcript_input_generation
+ == Some(self.realtime_conversation.input_generation)
+ {
+ self.realtime_conversation.latest_input_was_voice = true;
+ }
+ if self.realtime_conversation.transcript_role.as_deref() != Some(role.as_str()) {
+ let previous = self.realtime_conversation.transcript_role.take();
+ let previous_text = std::mem::take(&mut self.realtime_conversation.transcript);
+ let saved = self.realtime_conversation.interleaved_transcript.take();
+ self.realtime_conversation.transcript = match saved {
+ Some((saved_role, saved_text)) if saved_role == role => saved_text,
+ _ => String::new(),
+ };
+ if let Some(previous_role) = previous {
+ self.realtime_conversation.interleaved_transcript =
+ Some((previous_role, previous_text));
+ }
+ self.realtime_conversation.transcript_role = Some(role);
+ }
+ self.realtime_conversation.transcript.push_str(&delta);
+ if self.realtime_conversation.transcript.len() > MAX_TRANSCRIPT_BYTES {
+ let mut start = self.realtime_conversation.transcript.len() - MAX_TRANSCRIPT_BYTES;
+ while !self
+ .realtime_conversation
+ .transcript
+ .is_char_boundary(start)
+ {
+ start += 1;
+ }
+ self.realtime_conversation.transcript.drain(..start);
+ }
+ let role = self
+ .realtime_conversation
+ .transcript_role
+ .as_deref()
+ .unwrap_or("");
+ self.realtime_conversation.live_transcript_cell = Some(
+ self.realtime_transcript_history_cell(role, &self.realtime_conversation.transcript),
+ );
+ self.bump_active_cell_revision();
+ self.request_redraw();
+ }
+
+ pub(super) fn on_realtime_transcript_done(&mut self, role: String, mut text: String) {
+ if matches!(
+ self.realtime_conversation.phase,
+ RealtimeConversationPhase::Inactive | RealtimeConversationPhase::Stopping
+ ) {
+ if self.realtime_conversation.phase == RealtimeConversationPhase::Inactive
+ && !self.realtime_conversation.recover_late_transcripts
+ {
+ return;
+ }
+ if self.realtime_conversation.transcript_role.as_deref() == Some(role.as_str()) {
+ self.realtime_conversation.transcript_role = None;
+ self.realtime_conversation.transcript.clear();
+ self.realtime_conversation.live_transcript_cell = None;
+ self.bump_active_cell_revision();
+ } else if self
+ .realtime_conversation
+ .interleaved_transcript
+ .as_ref()
+ .is_some_and(|(saved_role, _)| saved_role == &role)
+ {
+ self.realtime_conversation.interleaved_transcript = None;
+ }
+ if text.trim().is_empty() {
+ if let Some(index) = self
+ .realtime_conversation
+ .accepted_transcripts
+ .iter()
+ .rposition(|record| record.role == role && !record.complete)
+ {
+ self.realtime_conversation
+ .accepted_transcripts
+ .remove(index);
+ }
+ return;
+ }
+ if text.len() > MAX_TRANSCRIPT_BYTES {
+ let mut end = MAX_TRANSCRIPT_BYTES;
+ while !text.is_char_boundary(end) {
+ end -= 1;
+ }
+ text.truncate(end);
+ }
+ let display_text = if let Some(partial) = self
+ .realtime_conversation
+ .accepted_transcripts
+ .iter_mut()
+ .rev()
+ .find(|record| record.role == role && !record.complete)
+ {
+ partial.text = text.clone();
+ partial.complete = true;
+ text.clone()
+ } else {
+ // A separate final can share a prefix with an earlier caption.
+ // Show it in full rather than guessing it extends that caption.
+ if self.realtime_conversation.accepted_transcripts.len()
+ >= MAX_REPLAY_TRANSCRIPT_CELLS
+ {
+ self.realtime_conversation.accepted_transcripts.pop_front();
+ }
+ self.realtime_conversation.accepted_transcripts.push_back(
+ RealtimeTranscriptRecord {
+ role: role.clone(),
+ text: text.clone(),
+ complete: true,
+ },
+ );
+ text.clone()
+ };
+ if display_text.is_empty() {
+ return;
+ }
+ if self.realtime_conversation.pending_history_cells.len() >= MAX_REPLAY_TRANSCRIPT_CELLS
+ {
+ self.realtime_conversation.pending_history_cells.pop_front();
+ }
+ let cell = self.realtime_transcript_history_cell(&role, &display_text);
+ self.realtime_conversation
+ .pending_history_cells
+ .push_back(cell);
+ self.flush_realtime_transcript_history();
+ return;
+ }
+ if !matches!(
+ self.realtime_conversation.phase,
+ RealtimeConversationPhase::Starting | RealtimeConversationPhase::Active
+ ) {
+ return;
+ }
+ let has_text = !text.trim().is_empty();
+ if role == "assistant" && has_text {
+ // appendSpeech's RPC acknowledgement only means the request was
+ // queued. A subsequent assistant caption is the closest signal
+ // available here that its answer reached the voice session. The
+ // protocol does not identify which speech request produced it.
+ // A done-only caption has no generation owner, so it cannot retire
+ // a newer answer's fallback.
+ if self
+ .realtime_conversation
+ .assistant_caption_started_after_speech_queue
+ && let Some(caption_generation) =
+ self.realtime_conversation.assistant_transcript_generation
+ && let Some(delivery) =
+ self.realtime_conversation
+ .pending_speech
+ .iter_mut()
+ .find(|delivery| {
+ !delivery.captioned
+ && delivery.input_generation == caption_generation
+ && delivery.state != PendingSpeechState::AwaitingTurn
+ && matches!(&delivery.item, ThreadItem::AgentMessage { text: answer, .. }
+ if answer.trim().strip_prefix("[FINAL]").unwrap_or(answer.trim())
+ .split_whitespace().eq(text.split_whitespace()))
+ })
+ {
+ delivery.captioned = true;
+ }
+ self.realtime_conversation
+ .pending_speech
+ .retain(|delivery| {
+ !delivery.captioned || matches!(delivery.state, PendingSpeechState::Queued(_))
+ });
+ }
+ if has_text
+ && self
+ .realtime_conversation
+ .transcript_input_generation
+ .is_none()
+ {
+ self.resume_realtime_speaker_for(&role, &text);
+ }
+ // Check ownership before retiring the old output: its final chunk must not unmute it.
+ if role == "assistant" {
+ self.realtime_conversation.assistant_transcript_generation = None;
+ self.realtime_conversation
+ .assistant_caption_started_after_speech_queue = false;
+ }
+ let voice_input_fingerprint = (role == "user").then(|| realtime_input_fingerprint(&text));
+ if role == "user" {
+ match self
+ .realtime_conversation
+ .transcript_input_generation
+ .take()
+ {
+ Some(generation) => {
+ if generation == self.realtime_conversation.input_generation {
+ self.realtime_conversation.latest_input_was_voice = has_text;
+ if !has_text {
+ self.release_realtime_speaker();
+ }
+ }
+ }
+ None => {
+ let old_superseded_transcript =
+ !self.realtime_conversation.latest_input_was_voice
+ && self
+ .realtime_conversation
+ .latest_voice_input_fingerprint
+ .is_some_and(|latest| Some(latest) == voice_input_fingerprint);
+ if old_superseded_transcript || !has_text {
+ return;
+ }
+ self.realtime_conversation.input_generation = self
+ .realtime_conversation
+ .input_generation
+ .wrapping_add(/*rhs*/ 1);
+ self.realtime_conversation.latest_input_was_voice = true;
+ self.suppress_realtime_speaker();
+ }
+ }
+ }
+ if self.realtime_conversation.transcript_role.as_deref() == Some(role.as_str()) {
+ self.realtime_conversation.transcript.clear();
+ self.realtime_conversation.transcript_role = None;
+ self.realtime_conversation.live_transcript_cell = None;
+ self.bump_active_cell_revision();
+ } else if self
+ .realtime_conversation
+ .interleaved_transcript
+ .as_ref()
+ .is_some_and(|(saved_role, _)| saved_role == &role)
+ {
+ self.realtime_conversation.interleaved_transcript = None;
+ }
+ if text.len() > MAX_TRANSCRIPT_BYTES {
+ let mut end = MAX_TRANSCRIPT_BYTES;
+ while !text.is_char_boundary(end) {
+ end -= 1;
+ }
+ text.truncate(end);
+ }
+ if role == "user" {
+ self.realtime_conversation.latest_voice_input_fingerprint = voice_input_fingerprint;
+ }
+ if !text.trim().is_empty() {
+ while self.realtime_conversation.accepted_transcripts.len()
+ >= MAX_PENDING_TRANSCRIPT_CELLS
+ {
+ self.realtime_conversation.accepted_transcripts.pop_front();
+ }
+ self.realtime_conversation
+ .accepted_transcripts
+ .push_back(RealtimeTranscriptRecord {
+ role: role.clone(),
+ text: text.clone(),
+ complete: true,
+ });
+ while self.realtime_conversation.pending_history_cells.len()
+ >= MAX_PENDING_TRANSCRIPT_CELLS
+ {
+ self.realtime_conversation.pending_history_cells.pop_front();
+ tracing::warn!("dropping the oldest deferred voice transcript");
+ }
+ let cell = self.realtime_transcript_history_cell(&role, &text);
+ self.realtime_conversation
+ .pending_history_cells
+ .push_back(cell);
+ self.flush_realtime_transcript_history();
+ }
+ }
+
+ fn finish_realtime_partial_transcripts(&mut self) {
+ if let Some((role, text)) = self.realtime_conversation.interleaved_transcript.take() {
+ self.on_realtime_transcript_done(role, text);
+ }
+ if let Some(role) = self.realtime_conversation.transcript_role.clone() {
+ let text = std::mem::take(&mut self.realtime_conversation.transcript);
+ self.on_realtime_transcript_done(role, text);
+ }
+ }
+
+ fn realtime_transcript_history_cell(&self, role: &str, text: &str) -> Box<dyn HistoryCell> {
+ if role == "user" {
+ Box::new(history_cell::new_user_prompt(
+ text.to_string(),
+ Vec::new(),
+ Vec::new(),
+ Vec::new(),
+ ))
+ } else {
+ Box::new(
+ history_cell::AgentMarkdownCell::new_with_inline_visualizations(
+ text.to_string(),
+ &self.config.cwd,
+ /*inline_visualization_context*/ None,
+ ),
+ )
+ }
+ }
+
+ pub(super) fn flush_realtime_transcript_history(&mut self) {
+ if self.stream_controller.is_some()
+ || self.plan_stream_controller.is_some()
+ || self.pending_stream_consolidations > 0
+ {
+ return;
+ }
+ while let Some(cell) = self.realtime_conversation.pending_history_cells.pop_front() {
+ self.add_boxed_history(cell);
+ }
+ }
+
+ pub(crate) fn on_realtime_error(&mut self, message: String) {
+ if self.realtime_conversation.phase == RealtimeConversationPhase::Inactive {
+ return;
+ }
+ self.stop_realtime_conversation();
+ self.add_error_message(message);
+ }
+
+ pub(super) fn on_realtime_conversation_closed(&mut self, reason: Option<String>) {
+ if self.realtime_conversation.phase == RealtimeConversationPhase::Inactive {
+ if self.realtime_conversation.recover_late_transcripts {
+ self.finish_realtime_partial_transcripts();
+ self.realtime_conversation.recover_late_transcripts = false;
+ }
+ return;
+ }
+ let retry_after_stop =
+ self.realtime_conversation.startup_retry == StartupRetry::WaitingForStop;
+ let retry_after_early_close = self.realtime_conversation.phase
+ == RealtimeConversationPhase::Starting
+ && self.realtime_conversation.startup_retry == StartupRetry::Available
+ && reason.as_deref() == Some("transport_closed");
+ if self.realtime_conversation.phase == RealtimeConversationPhase::Stopping
+ && reason.as_deref() != Some("requested")
+ {
+ return;
+ }
+ // A transport close may arrive without final transcript events. Recover
+ // answers before flushing partial captions, which do not prove delivery.
+ self.restore_all_undelivered_realtime_speech();
+ self.finish_realtime_partial_transcripts();
+ let muted = self.realtime_conversation.microphone_muted;
+ let thread_id = self.realtime_conversation.thread_id;
+ self.reset_realtime_conversation();
+ if (retry_after_stop || retry_after_early_close)
+ && let Some(thread_id) = thread_id.filter(|id| Some(*id) == self.thread_id())
+ {
+ // The old backend is closed. A late peer result belongs to its attempt ID.
+ self.realtime_conversation.startup_retry = StartupRetry::Used;
+ self.realtime_conversation.microphone_muted = muted;
+ if retry_after_early_close {
+ self.add_info_message(
+ "Voice connection closed during startup. Retrying once.".into(),
+ /*hint*/ None,
+ );
+ }
+ self.start_realtime_conversation(thread_id);
+ return;
+ }
+ if let Some(reason) = reason
+ && reason != "error"
+ {
+ self.add_info_message(
+ format!("Voice conversation ended: {reason}"),
+ /*hint*/ None,
+ );
+ }
+ self.request_redraw();
+ }
+
+ pub(crate) fn reset_realtime_conversation(&mut self) -> Option<ThreadId> {
+ self.restore_all_undelivered_realtime_speech();
+ // A delegated item is hidden until the turn completes so it can be spoken.
+ // If voice ends first, let the later turn completion render it normally.
+ if self
+ .transcript
+ .last_completed_agent_message
+ .as_ref()
+ .is_some_and(|completed_item| {
+ matches!(
+ self.realtime_conversation.agent_items.get(completed_item),
+ Some(RealtimeAgentItemOrigin::Delegated {
+ completed: true,
+ ..
+ })
+ )
+ })
+ {
+ self.transcript.last_completed_agent_message = None;
+ }
+ // The helper may already have exited while app-server still owns the
+ // voice session. An acknowledged backend start still needs a stop RPC.
+ let backend_thread_id = if self.realtime_conversation.handle.is_some()
+ || self.realtime_conversation.backend_started
+ {
+ self.realtime_conversation.thread_id
+ } else {
+ None
+ };
+ if let Some(abort) = self.realtime_conversation.startup_abort.take() {
+ abort.abort();
+ }
+ if let Some(handle) = self.realtime_conversation.handle.take() {
+ handle.close();
+ }
+ // Direct resets (for example, switching to a resumed thread) do not
+ // pass through the normal stop/close transcript flush.
+ let partials = [
+ self.realtime_conversation.interleaved_transcript.take(),
+ self.realtime_conversation
+ .transcript_role
+ .take()
+ .map(|role| {
+ (
+ role,
+ std::mem::take(&mut self.realtime_conversation.transcript),
+ )
+ }),
+ ];
+ for (role, text) in partials.into_iter().flatten() {
+ if text.trim().is_empty() {
+ continue;
+ }
+ if self.realtime_conversation.accepted_transcripts.len() >= MAX_REPLAY_TRANSCRIPT_CELLS
+ {
+ self.realtime_conversation.accepted_transcripts.pop_front();
+ }
+ if self.realtime_conversation.pending_history_cells.len() >= MAX_REPLAY_TRANSCRIPT_CELLS
+ {
+ self.realtime_conversation.pending_history_cells.pop_front();
+ }
+ let cell = self.realtime_transcript_history_cell(&role, &text);
+ self.realtime_conversation
+ .pending_history_cells
+ .push_back(cell);
+ self.realtime_conversation
+ .accepted_transcripts
+ .push_back(RealtimeTranscriptRecord {
+ role,
+ text,
+ complete: true,
+ });
+ }
+ let pending_history_cells =
+ std::mem::take(&mut self.realtime_conversation.pending_history_cells);
+ let accepted_transcripts =
+ std::mem::take(&mut self.realtime_conversation.accepted_transcripts);
+ let delegated_reasoning_turns =
+ std::mem::take(&mut self.realtime_conversation.delegated_reasoning_turns);
+ let had_live_transcript = self.realtime_conversation.live_transcript_cell.is_some();
+ self.realtime_conversation = RealtimeConversationUiState {
+ attempt_id: self.realtime_conversation.attempt_id,
+ pending_history_cells,
+ accepted_transcripts,
+ delegated_reasoning_turns,
+ ..RealtimeConversationUiState::default()
+ };
+ if had_live_transcript {
+ self.bump_active_cell_revision();
+ }
+ self.set_footer_hint_override(/*items*/ None);
+ self.flush_realtime_transcript_history();
+ backend_thread_id
+ }
+
+ pub(super) fn refresh_realtime_microphone_level(&mut self) {
+ if !matches!(
+ self.realtime_conversation.phase,
+ RealtimeConversationPhase::Starting | RealtimeConversationPhase::Active
+ ) {
+ return;
+ }
+ let Some(handle) = self.realtime_conversation.handle.as_ref() else {
+ return;
+ };
+ if let Some(error) = handle.take_error() {
+ self.on_realtime_error(format!("Voice conversation failed: {error}"));
+ return;
+ }
+ if self.realtime_conversation.phase == RealtimeConversationPhase::Starting {
+ self.frame_requester
+ .schedule_frame_in(MICROPHONE_METER_INTERVAL);
+ return;
+ }
+ let microphone_level = if self.realtime_conversation.microphone_muted {
+ 0
+ } else {
+ audio_meter_level(handle.take_microphone_peak())
+ };
+ let speaker_level = audio_meter_level(handle.take_speaker_peak());
+ let mut changed = self.realtime_conversation.microphone_level != microphone_level
+ || self.realtime_conversation.speaker_level != speaker_level;
+ if speaker_level > 0 {
+ self.realtime_conversation.speaker_active_until =
+ Some(Instant::now() + SPEAKER_ACTIVITY_HOLD);
+ } else {
+ changed |= self
+ .realtime_conversation
+ .speaker_active_until
+ .take_if(|deadline| *deadline <= Instant::now())
+ .is_some();
+ }
+ self.realtime_conversation.microphone_level = microphone_level;
+ self.realtime_conversation.speaker_level = speaker_level;
+ if changed {
+ self.update_realtime_footer();
+ }
+ self.frame_requester
+ .schedule_frame_in(MICROPHONE_METER_INTERVAL);
+ }
+
+ fn update_realtime_footer(&mut self) {
+ let status = if self.realtime_conversation.phase == RealtimeConversationPhase::Starting {
+ "◌ connecting"
+ } else if self.realtime_conversation.microphone_muted {
+ "◌ muted"
+ } else if self.realtime_conversation.speaker_level > 0
+ || self
+ .realtime_conversation
+ .speaker_active_until
+ .is_some_and(|deadline| deadline > Instant::now())
+ {
+ "● speaking"
+ } else {
+ "● listening"
+ };
+ let mut hints = vec![
+ ("voice".to_string(), status.to_string()),
+ ("/voice".to_string(), "stop".to_string()),
+ ];
+ if self.realtime_conversation.phase == RealtimeConversationPhase::Active {
+ let microphone = if self.realtime_conversation.microphone_muted {
+ "off".to_string()
+ } else {
+ audio_meter_bar(self.realtime_conversation.microphone_level)
+ };
+ let speaker = audio_meter_bar(self.realtime_conversation.speaker_level);
+ hints.push(("mic".to_string(), format!("{microphone} codex {speaker}")));
+ }
+ hints.push(("/voice".to_string(), "mute".to_string()));
+ self.set_footer_hint_override(Some(hints));
+ }
+}
+
+fn audio_meter_level(peak: u16) -> usize {
+ usize::from(peak.saturating_sub(AUDIO_METER_NOISE_FLOOR))
+ .saturating_mul(AUDIO_METER_SEGMENTS)
+ .div_ceil(usize::from(
+ AUDIO_METER_FULL_SCALE - AUDIO_METER_NOISE_FLOOR,
+ ))
+ .min(AUDIO_METER_SEGMENTS)
+}
+
+fn audio_meter_bar(level: usize) -> String {
+ let level = level.min(AUDIO_METER_SEGMENTS);
+ format!(
+ "{}{}",
+ "█".repeat(level),
+ "░".repeat(AUDIO_METER_SEGMENTS - level)
+ )
+}
diff --git a/codex-rs/tui/src/chatwidget/realtime_tests.rs b/codex-rs/tui/src/chatwidget/realtime_tests.rs
new file mode 100644
index 0000000000..8c194c47a2
--- /dev/null
+++ b/codex-rs/tui/src/chatwidget/realtime_tests.rs
@@ -0,0 +1,107 @@
+//! Focused visual coverage and shared setup for voice behavior tests.
+
+use super::RealtimeConversationPhase;
+use crate::app_event::AppEvent;
+use crate::chatwidget::ChatWidget;
+use crate::chatwidget::tests::make_chatwidget_manual_with_sender;
+use crate::chatwidget::tests::render_bottom_popup;
+use codex_protocol::ThreadId;
+
+pub(crate) fn activate_voice_for_thread(chat: &mut ChatWidget, thread_id: ThreadId) {
+ chat.thread_id = Some(thread_id);
+ chat.realtime_conversation.phase = RealtimeConversationPhase::Active;
+ chat.realtime_conversation.thread_id = Some(thread_id);
+ chat.realtime_conversation.backend_started = true;
+ chat.realtime_conversation.latest_input_was_voice = true;
+}
+
+#[tokio::test]
+async fn enabling_voice_on_an_open_thread_snapshots_the_new_thread_notice() {
+ let (mut chat, _sender, mut events, _ops) = make_chatwidget_manual_with_sender().await;
+ chat.set_feature_enabled(
+ codex_features::Feature::RealtimeConversation,
+ /*enabled*/ true,
+ );
+ let rendered = std::iter::from_fn(|| events.try_recv().ok())
+ .filter_map(|event| match event {
+ AppEvent::InsertHistoryCell(cell) => Some(
+ cell.display_lines(/*width*/ 80)
+ .into_iter()
+ .map(|line| line.to_string())
+ .collect::<Vec<_>>()
+ .join("\n"),
+ ),
+ _ => None,
+ })
+ .collect::<Vec<_>>()
+ .join("\n");
+ insta::assert_snapshot!(rendered, @"• Voice conversations will be available in new threads.");
+}
+
+#[tokio::test]
+async fn voice_footer_renders_the_main_conversation_states() {
+ let (mut chat, _sender, _events, _ops) = make_chatwidget_manual_with_sender().await;
+ let mut states = Vec::new();
+
+ for (label, phase, muted, level, speaker_level, role, transcript) in [
+ (
+ "connecting",
+ RealtimeConversationPhase::Starting,
+ false,
+ 0,
+ 0,
+ None,
+ "",
+ ),
+ (
+ "listening",
+ RealtimeConversationPhase::Active,
+ false,
+ 0,
+ 0,
+ None,
+ "",
+ ),
+ (
+ "speaking",
+ RealtimeConversationPhase::Active,
+ false,
+ 4,
+ 5,
+ None,
+ "",
+ ),
+ (
+ "muted",
+ RealtimeConversationPhase::Active,
+ true,
+ 4,
+ 4,
+ None,
+ "",
+ ),
+ (
+ "transcript",
+ RealtimeConversationPhase::Active,
+ false,
+ 2,
+ 2,
+ Some("assistant"),
+ "Hello there",
+ ),
+ ] {
+ chat.realtime_conversation.phase = phase;
+ chat.realtime_conversation.microphone_muted = muted;
+ chat.realtime_conversation.microphone_level = level;
+ chat.realtime_conversation.speaker_level = speaker_level;
+ chat.realtime_conversation.transcript_role = role.map(str::to_string);
+ chat.realtime_conversation.transcript = transcript.to_string();
+ chat.update_realtime_footer();
+ states.push(format!(
+ "{label}:\n{}",
+ render_bottom_popup(&chat, /*width*/ 80)
+ ));
+ }
+
+ insta::assert_snapshot!(states.join("\n\n"));
+}
diff --git a/codex-rs/tui/src/chatwidget/reconnect.rs b/codex-rs/tui/src/chatwidget/reconnect.rs
index 1de24dcb74..63d5d28f59 100644
--- a/codex-rs/tui/src/chatwidget/reconnect.rs
+++ b/codex-rs/tui/src/chatwidget/reconnect.rs
@@ -4,6 +4,9 @@ use super::*;
impl ChatWidget {
pub(crate) fn pause_for_disconnect(&mut self) {
+ // The app-server transport can fail while the separate WebRTC helper
+ // still sends microphone audio. Retire local media before showing offline UI.
+ let _ = self.reset_realtime_conversation();
if let Some(questions) = &mut self.bottom_pane.questions {
questions.delivery_enabled = false;
}
diff --git a/codex-rs/tui/src/chatwidget/rendering.rs b/codex-rs/tui/src/chatwidget/rendering.rs
index 61a5403d27..8163ae2549 100644
--- a/codex-rs/tui/src/chatwidget/rendering.rs
+++ b/codex-rs/tui/src/chatwidget/rendering.rs
@@ -158,6 +158,17 @@ impl ChatWidget {
};
let mut flex = FlexRenderable::new();
flex.push(/*flex*/ 1, active_cell_renderable);
+ if let Some(cell) = self.realtime_conversation.live_transcript_cell.as_ref() {
+ flex.push(
+ /*flex*/ 1,
+ RenderableItem::Owned(Box::new(TranscriptAreaRenderable {
+ child: cell.as_ref(),
+ top: 1,
+ right: active_cell_right_reserve,
+ persistent_layout: None,
+ })),
+ );
+ }
if let Some(cell) = self.pending_token_activity_output() {
flex.push(
/*flex*/ 1,
diff --git a/codex-rs/tui/src/chatwidget/replay.rs b/codex-rs/tui/src/chatwidget/replay.rs
index 18ad2d01f3..a233ce20e0 100644
--- a/codex-rs/tui/src/chatwidget/replay.rs
+++ b/codex-rs/tui/src/chatwidget/replay.rs
@@ -46,13 +46,31 @@ impl ChatWidget {
completed_at,
duration_ms,
} = turn;
+ let delegated = items.iter().any(|item| {
+ matches!(item, ThreadItem::UserMessage { content, .. }
+ if realtime::realtime_delegation_input(content).is_some())
+ });
if matches!(status, TurnStatus::InProgress) {
+ if delegated {
+ self.remember_realtime_delegated_reasoning_turn(&turn_id);
+ }
self.warning_display_state.startup_complete = true;
self.turn_lifecycle.last_turn_id = Some(turn_id.clone());
self.last_non_retry_error = None;
self.on_task_started();
}
+ let mut replaying_delegation = false;
for item in items {
+ if matches!(&item, ThreadItem::UserMessage { content, .. }
+ if realtime::realtime_delegation_input(content).is_some())
+ {
+ replaying_delegation = true;
+ }
+ // Voice can steer a typed turn already in progress. Its earlier
+ // commentary and reasoning still belong to the typed request.
+ if replaying_delegation && realtime::is_private_realtime_agent_item(&item) {
+ continue;
+ }
if hidden_nested_review_turn && matches!(item, ThreadItem::UserMessage { .. }) {
continue;
}
@@ -122,7 +140,12 @@ impl ChatWidget {
ThreadItem::UserMessage {
content, client_id, ..
} => {
- self.on_committed_user_message(&content, client_id.as_deref(), from_replay);
+ self.on_committed_user_message(
+ &content,
+ client_id.as_deref(),
+ from_replay,
+ &turn_id,
+ );
}
ThreadItem::AgentMessage {
id,
@@ -133,6 +156,20 @@ impl ChatWidget {
questions,
..
} => {
+ if self.complete_realtime_delegated_agent_item(
+ &turn_id,
+ &ThreadItem::AgentMessage {
+ id: id.clone(),
+ text: text.clone(),
+ phase: phase.clone(),
+ memory_citation: memory_citation.clone(),
+ delivery,
+ questions: questions.clone(),
+ },
+ from_replay,
+ ) {
+ return;
+ }
self.on_agent_message_item_completed(
AgentMessageItem {
id,
diff --git a/codex-rs/tui/src/chatwidget/session_flow.rs b/codex-rs/tui/src/chatwidget/session_flow.rs
index c29a574643..a24e3695f0 100644
--- a/codex-rs/tui/src/chatwidget/session_flow.rs
+++ b/codex-rs/tui/src/chatwidget/session_flow.rs
@@ -24,6 +24,11 @@ impl ChatWidget {
let connector_scope_changed = previous_thread_id != Some(session.thread_id)
|| self.config.cwd.as_path() != session.cwd.as_path();
self.thread_id = Some(session.thread_id);
+ self.realtime_conversation_available_for_thread =
+ self.config.features.enabled(Feature::RealtimeConversation)
+ && codex_realtime_webrtc::RealtimeWebrtcSession::is_supported();
+ self.bottom_pane
+ .set_voice_command_enabled(self.realtime_conversation_available_for_thread);
self.bottom_pane
.set_queue_submissions(/*queue_submissions*/ false);
if previous_thread_id != self.thread_id {
diff --git a/codex-rs/tui/src/chatwidget/settings.rs b/codex-rs/tui/src/chatwidget/settings.rs
index 224feab67a..c630d43f41 100644
--- a/codex-rs/tui/src/chatwidget/settings.rs
+++ b/codex-rs/tui/src/chatwidget/settings.rs
@@ -100,6 +100,21 @@ impl ChatWidget {
self.update_collaboration_mode_indicator();
}
}
+ if feature == Feature::RealtimeConversation && !enabled {
+ self.realtime_conversation_available_for_thread = false;
+ self.bottom_pane
+ .set_voice_command_enabled(/*enabled*/ false);
+ self.stop_realtime_conversation();
+ }
+ if feature == Feature::RealtimeConversation
+ && enabled
+ && !self.realtime_conversation_available_for_thread
+ {
+ self.add_info_message(
+ "Voice conversations will be available in new threads.".into(),
+ /*hint*/ None,
+ );
+ }
if feature == Feature::MentionsV2 {
self.sync_mentions_v2_enabled();
}
diff --git a/codex-rs/tui/src/chatwidget/slash_dispatch.rs b/codex-rs/tui/src/chatwidget/slash_dispatch.rs
index b8db07ee74..453aad4022 100644
--- a/codex-rs/tui/src/chatwidget/slash_dispatch.rs
+++ b/codex-rs/tui/src/chatwidget/slash_dispatch.rs
@@ -333,6 +333,9 @@ impl ChatWidget {
);
}
}
+ SlashCommand::Voice => {
+ self.toggle_realtime_conversation();
+ }
SlashCommand::Side | SlashCommand::Btw => {
self.request_empty_side_conversation(cmd);
}
@@ -757,6 +760,11 @@ impl ChatWidget {
}
}
}
+ SlashCommand::Voice => match trimmed.to_ascii_lowercase().as_str() {
+ "mute" => self.toggle_realtime_microphone(),
+ "stop" => self.stop_realtime_conversation(),
+ _ => self.add_error_message("Usage: /voice [mute|stop]".to_string()),
+ },
SlashCommand::Ide => {
self.handle_ide_command_args(trimmed);
}
@@ -1144,6 +1152,7 @@ impl ChatWidget {
goal_command_enabled: self.config.features.enabled(Feature::Goals),
service_tier_commands_enabled: self.fast_mode_enabled(),
personality_command_enabled: self.config.features.enabled(Feature::Personality),
+ voice_command_enabled: self.realtime_conversation_available_for_thread,
worktrees_enabled: self.config.features.enabled(Feature::Worktrees)
&& self.local_worktree_operations,
allow_elevate_sandbox,
@@ -1183,6 +1192,7 @@ impl ChatWidget {
| SlashCommand::Diff
| SlashCommand::App
| SlashCommand::Rename
+ | SlashCommand::Voice
| SlashCommand::Recap
| SlashCommand::TestApproval => QueueDrain::Continue,
SlashCommand::Cd => match self.thread_id {
diff --git a/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__realtime__tests__voice_footer_renders_the_main_conversation_states.snap b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__realtime__tests__voice_footer_renders_the_main_conversation_states.snap
new file mode 100644
index 0000000000..cb205e97df
--- /dev/null
+++ b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__realtime__tests__voice_footer_renders_the_main_conversation_states.snap
@@ -0,0 +1,29 @@
+---
+source: tui/src/chatwidget/realtime_tests.rs
+assertion_line: 176
+expression: "states.join(\"\\n\\n\")"
+---
+connecting:
+› Ask Codex to do anything
+
+ voice ◌ connecting /voice stop /voice mute
+
+listening:
+› Ask Codex to do anything
+
+ voice ● listening /voice stop mic ░░░░░ codex ░░░░░ /voice mute
+
+speaking:
+› Ask Codex to do anything
+
+ voice ● speaking /voice stop mic ████░ codex █████ /voice mute
+
+muted:
+› Ask Codex to do anything
+
+ voice ◌ muted /voice stop mic off codex ████░ /voice mute
+
+transcript:
+› Ask Codex to do anything
+
+ voice ● speaking /voice stop mic ██░░░ codex ██░░░ /voice mute
diff --git a/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__experimental_features_available_popup.snap b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__experimental_features_available_popup.snap
new file mode 100644
index 0000000000..8ff8096740
--- /dev/null
+++ b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__experimental_features_available_popup.snap
@@ -0,0 +1,16 @@
+---
+source: tui/src/chatwidget/tests/popups_and_settings.rs
+assertion_line: 3142
+expression: popup
+---
+ Experimental features
+ Checked features are configured on. Some experimental features take effect
+ only in new tasks or after restarting the Codex server.
+
+› [ ] Network proxy Apply network proxy restrictions to
+ sandboxed sessions that already have
+ network access.
+ [ ] Prevent sleep while running Keep your computer awake while Codex is
+ running a thread.
+
+ Press space to select or enter to save
diff --git a/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__voice_live_transcript_and_stream.snap b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__voice_live_transcript_and_stream.snap
new file mode 100644
index 0000000000..61937b7844
--- /dev/null
+++ b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__voice_live_transcript_and_stream.snap
@@ -0,0 +1,16 @@
+---
+source: tui/src/chatwidget/tests/status_and_layout.rs
+assertion_line: 34
+expression: rendered
+---
+" "
+"• Agent answer arriving "
+" "
+" "
+"› pick a number "
+" "
+" "
+" "
+"› Ask Codex to do anything "
+" "
+" voice ● listening /voice stop mic ░░░░░ codex ░░░░░ "
diff --git a/codex-rs/tui/src/chatwidget/tests/misalignment_policy_tests.rs b/codex-rs/tui/src/chatwidget/tests/misalignment_policy_tests.rs
index 908af55709..4516460017 100644
--- a/codex-rs/tui/src/chatwidget/tests/misalignment_policy_tests.rs
+++ b/codex-rs/tui/src/chatwidget/tests/misalignment_policy_tests.rs
@@ -1,8 +1,51 @@
use super::*;
+use crate::chatwidget::realtime::tests::activate_voice_for_thread;
use pretty_assertions::assert_eq;
const ERROR_MESSAGE: &str = "Responses API returned misalignment_policy_violation";
+#[tokio::test]
+async fn misalignment_precaution_retires_voice_and_keeps_late_turn_blocked() {
+ let (mut chat, _events, mut ops) = make_chatwidget_manual(/*model_override*/ None).await;
+ let thread_id = ThreadId::new();
+ chat.thread_id = Some(thread_id);
+ handle_turn_started(&mut chat, "unsafe-turn");
+ activate_voice_for_thread(&mut chat, thread_id);
+
+ handle_error(
+ &mut chat,
+ ERROR_MESSAGE,
+ Some(CodexErrorInfo::MisalignmentPolicyViolation),
+ );
+
+ assert!(!chat.realtime_conversation_is_running());
+ assert_matches!(
+ ops.try_recv(),
+ Ok(Op::RealtimeConversationStop { thread_id: stopped }) if stopped == thread_id
+ );
+ assert!(ops.try_recv().is_err());
+ assert!(chat.has_misalignment_policy_violation());
+ assert!(!chat.bottom_pane.composer_input_enabled());
+ assert!(chat.submit_op(Op::RealtimeConversationStop { thread_id }));
+ assert_matches!(ops.try_recv(), Ok(Op::RealtimeConversationStop { .. }));
+
+ // A handoff already in flight may still produce a TurnStarted notification.
+ // It must not dismiss the precaution or restart local capture.
+ handle_turn_started(&mut chat, "late-voice-turn");
+ assert!(chat.has_misalignment_policy_violation());
+ assert!(!chat.bottom_pane.composer_input_enabled());
+ chat.toggle_realtime_conversation();
+ assert!(!chat.realtime_conversation_is_running());
+ assert!(ops.try_recv().is_err());
+
+ chat.clear_misalignment_for_new_turn(
+ "acknowledged-turn",
+ MisalignmentTurnSource::AcknowledgedContinuation,
+ );
+ assert!(!chat.has_misalignment_policy_violation());
+ assert!(chat.bottom_pane.composer_input_enabled());
+}
+
#[tokio::test]
async fn misalignment_policy_failure_stops_the_thread_and_renders_once() {
let (mut chat, mut rx, mut op_rx) = make_chatwidget_manual(/*model_override*/ None).await;
diff --git a/codex-rs/tui/src/chatwidget/tests/popups_and_settings.rs b/codex-rs/tui/src/chatwidget/tests/popups_and_settings.rs
index e7617cc3c9..1260a61fbc 100644
--- a/codex-rs/tui/src/chatwidget/tests/popups_and_settings.rs
+++ b/codex-rs/tui/src/chatwidget/tests/popups_and_settings.rs
@@ -3175,6 +3175,35 @@ async fn experimental_popup_loading_snapshot() {
assert!(rx.try_recv().is_err());
}
+#[tokio::test]
+async fn experimental_popup_available_snapshot() {
+ let (mut chat, mut rx, _op_rx) = make_chatwidget_manual(/*model_override*/ None).await;
+ chat.thread_id = Some(ThreadId::new());
+ chat.open_experimental_popup();
+ let AppEvent::FetchExperimentalFeatures { response_tx, .. } = rx.try_recv().unwrap() else {
+ panic!("expected experimental discovery request");
+ };
+ let features = [
+ ("network_proxy", "Network proxy", "Apply network proxy restrictions to sandboxed sessions that already have network access."),
+ ("prevent_idle_sleep", "Prevent sleep while running", "Keep your computer awake while Codex is running a thread."),
+ ]
+ .into_iter()
+ .map(|(name, display_name, description)| codex_app_server_protocol::ExperimentalFeature {
+ name: name.to_string(),
+ stage: codex_app_server_protocol::ExperimentalFeatureStage::Beta,
+ display_name: Some(display_name.to_string()),
+ description: Some(description.to_string()),
+ announcement: None,
+ enabled: false,
+ default_enabled: false,
+ })
+ .collect();
+ response_tx.send(Ok(features)).unwrap();
+ chat.pre_draw_tick();
+ let popup = render_bottom_popup(&chat, /*width*/ 80);
+ assert_chatwidget_snapshot!("experimental_features_available_popup", popup);
+}
+
#[tokio::test]
async fn feature_enable_prompts_snapshot() {
let (mut chat, mut rx, _op_rx) = make_chatwidget_manual(/*model_override*/ None).await;
diff --git a/codex-rs/tui/src/chatwidget/tests/status_and_layout.rs b/codex-rs/tui/src/chatwidget/tests/status_and_layout.rs
index 4ae4c253f3..8de9141a6f 100644
--- a/codex-rs/tui/src/chatwidget/tests/status_and_layout.rs
+++ b/codex-rs/tui/src/chatwidget/tests/status_and_layout.rs
@@ -3,12 +3,37 @@ use crate::bottom_pane::goal_status_indicator_line;
use crate::chatwidget::ThreadUsageOutcome;
use crate::chatwidget::rate_limits::NUDGE_MODEL_SLUG;
use crate::chatwidget::rate_limits::get_limits_duration;
+use crate::chatwidget::realtime::tests::activate_voice_for_thread;
use codex_app_server_protocol::SpendControlLimitSnapshot;
use codex_app_server_protocol::ThreadUsage;
use pretty_assertions::assert_eq;
+use ratatui::Terminal;
use ratatui::backend::TestBackend;
use serial_test::serial;
+#[tokio::test]
+async fn voice_live_transcript_renders_beside_the_streamed_cell() {
+ let (mut chat, _rx, _ops) = make_chatwidget_manual(/*model_override*/ None).await;
+ chat.config.animations = false;
+ activate_voice_for_thread(&mut chat, ThreadId::new());
+ chat.transcript.active_cell = Some(Box::new(history_cell::StreamingAgentTailCell::new(
+ vec![Line::from("Agent answer arriving").into()],
+ /*is_first_line*/ true,
+ )));
+ chat.on_realtime_transcript_delta("user".into(), "pick a number".into());
+
+ let width = 60;
+ let height = chat.desired_height(width);
+ let mut terminal = Terminal::new(TestBackend::new(width, height)).expect("create terminal");
+ terminal
+ .draw(|frame| chat.render(frame.area(), frame.buffer_mut()))
+ .expect("render live voice transcript");
+ let rendered = normalized_backend_snapshot(terminal.backend());
+ assert!(rendered.contains("Agent answer arriving"), "{rendered}");
+ assert!(rendered.contains("pick a number"), "{rendered}");
+ assert_chatwidget_snapshot!("voice_live_transcript_and_stream", rendered);
+}
+
fn enable_test_ambient_pet(chat: &mut ChatWidget) {
chat.set_pet_image_support_for_tests(crate::pets::PetImageSupport::Supported(
crate::pets::ImageProtocol::Kitty,
diff --git a/codex-rs/tui/src/chatwidget/turn_runtime.rs b/codex-rs/tui/src/chatwidget/turn_runtime.rs
index 18f51fff29..e52dc9118b 100644
--- a/codex-rs/tui/src/chatwidget/turn_runtime.rs
+++ b/codex-rs/tui/src/chatwidget/turn_runtime.rs
@@ -208,7 +208,14 @@ impl ChatWidget {
.current_goal_status
.as_ref()
.is_some_and(GoalStatusState::is_active);
- if !follow_up_started && !active_goal_continuing {
+ if !follow_up_started
+ && !active_goal_continuing
+ && !self
+ .turn_lifecycle
+ .last_turn_id
+ .as_deref()
+ .is_some_and(|turn_id| self.should_hide_realtime_delegation(turn_id))
+ {
self.notify(Notification::AgentTurnComplete {
response: notification_response,
});
diff --git a/codex-rs/tui/src/slash_command.rs b/codex-rs/tui/src/slash_command.rs
index ca83861bdb..93feadc318 100644
--- a/codex-rs/tui/src/slash_command.rs
+++ b/codex-rs/tui/src/slash_command.rs
@@ -41,6 +41,7 @@ pub enum SlashCommand {
Compact,
Recap,
Plan,
+ Voice,
Goal,
Agents,
Side,
@@ -130,6 +131,7 @@ impl SlashCommand {
}
SlashCommand::Personality => "choose a communication style for Codex",
SlashCommand::Plan => "switch to Plan mode",
+ SlashCommand::Voice => "start or stop a live voice conversation",
SlashCommand::Goal => "set or view the goal for a long-running task",
SlashCommand::Agents => "view and switch between all active agent sessions",
SlashCommand::MultiAgents => "switch between this session's subagents",
@@ -172,6 +174,7 @@ impl SlashCommand {
| SlashCommand::Fork
| SlashCommand::Plan
| SlashCommand::Goal
+ | SlashCommand::Voice
| SlashCommand::Ide
| SlashCommand::Keymap
| SlashCommand::Mcp
@@ -250,6 +253,7 @@ impl SlashCommand {
| SlashCommand::Stop
| SlashCommand::App
| SlashCommand::Goal
+ | SlashCommand::Voice
| SlashCommand::Mcp
| SlashCommand::Apps
| SlashCommand::Plugins
@@ -274,6 +278,7 @@ impl SlashCommand {
SlashCommand::SandboxReadRoot => cfg!(target_os = "windows"),
SlashCommand::Copy => !cfg!(target_os = "android"),
SlashCommand::App => cfg!(any(target_os = "macos", target_os = "windows")),
+ SlashCommand::Voice => true,
SlashCommand::Rollout | SlashCommand::TestApproval => cfg!(debug_assertions),
_ => true,
}
diff --git a/codex-rs/voice-host/src/main.rs b/codex-rs/voice-host/src/main.rs
index 3fc237eb89..20c252eb21 100644
--- a/codex-rs/voice-host/src/main.rs
+++ b/codex-rs/voice-host/src/main.rs
@@ -176,9 +176,15 @@ fn run(
let Some(peer) = transport.as_ref() else {
return Err(io::Error::other("voice transport not started"));
};
- executor
+ let outcome = executor
.block_on(peer.apply_answer(sdp.into_sdp()))
.map_err(io::Error::other)?;
+ if outcome == transport::AnswerOutcome::TimedOut {
+ // The client reaps this helper before considering a fresh negotiation.
+ output.write_all(&encode_frame(&Message::TransportTimedOut {})?)?;
+ output.flush()?;
+ return Ok(());
+ }
answered = true;
Message::TransportReady {}
}
@@ -207,6 +213,7 @@ fn run(
| Message::ApplyAnswer { .. }
| Message::Offer { .. }
| Message::TransportReady {}
+ | Message::TransportTimedOut {}
| Message::OpenDevices {}
| Message::DevicesOpened {}
| Message::AudioControlsApplied {}
diff --git a/codex-rs/voice-host/src/transport.rs b/codex-rs/voice-host/src/transport.rs
index 3b0646576d..e3769093a2 100644
--- a/codex-rs/voice-host/src/transport.rs
+++ b/codex-rs/voice-host/src/transport.rs
@@ -59,6 +59,12 @@ impl PeerConnectionEventHandler for Events {
}
}
+#[derive(Debug, PartialEq)]
+pub(crate) enum AnswerOutcome {
+ Ready,
+ TimedOut,
+}
+
pub(crate) struct Transport {
pub(crate) incoming: crate::incoming::Incoming,
pub(crate) audio: crate::audio_track::AudioTrack,
@@ -162,7 +168,7 @@ impl Transport {
.map_err(|_| "timed out gathering voice candidates")?
}
- pub(crate) async fn apply_answer(&self, sdp: String) -> Result<()> {
+ pub(crate) async fn apply_answer(&self, sdp: String) -> Result<AnswerOutcome> {
let answer = RTCSessionDescription::answer(sdp).map_err(|_| "invalid voice answer")?;
let parsed = answer.unmarshal().map_err(|_| "invalid voice answer")?;
let mut seen = HashSet::new();
@@ -213,7 +219,9 @@ impl Transport {
.map_err(|_| "voice event channel closed")
})
.await
- .map_err(|_| "timed out connecting voice peer")?
+ .map_or(Ok(AnswerOutcome::TimedOut), |result| {
+ result.map(|()| AnswerOutcome::Ready)
+ })
}
pub(crate) async fn close(&mut self) -> Result<()> {
diff --git a/codex-rs/voice-host/src/transport_tests.rs b/codex-rs/voice-host/src/transport_tests.rs
index 20eda40ad9..3646959820 100644
--- a/codex-rs/voice-host/src/transport_tests.rs
+++ b/codex-rs/voice-host/src/transport_tests.rs
@@ -86,10 +86,11 @@ async fn check_negotiation(runtime: Arc<dyn webrtc::runtime::Runtime>) {
for _ in candidates.len()..MAX_REMOTE_CANDIDATES {
answer.push_str(&format!("{}\r\n", candidates[0]));
}
- local
+ let outcome = local
.apply_answer(answer)
.await
.unwrap_or_else(|error| panic!("tcp={tcp}: {error}"));
+ assert_eq!(outcome, super::AnswerOutcome::Ready);
let channel = channels.recv().await.unwrap();
assert_eq!(
(