Lightweight coding agent that runs in your terminal
Rust
Keep voice sessions alive through mute and audio backlog (#44671)
Benjamin Carlsson28f43b0417ab
diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lockindex 39c1d3accb..572ffcd05b 100644--- a/codex-rs/Cargo.lock+++ b/codex-rs/Cargo.lock@@ -4309,6 +4309,7 @@ dependencies = ["serde","serde_json","tokio",+ "tracing",][[package]]diff --git a/codex-rs/realtime-webrtc/Cargo.toml b/codex-rs/realtime-webrtc/Cargo.tomlindex 9042544395..2d1b3b422f 100644--- a/codex-rs/realtime-webrtc/Cargo.toml+++ b/codex-rs/realtime-webrtc/Cargo.toml@@ -19,6 +19,7 @@ futures = { workspace = true, features = ["std"] }serde = { workspace = true, features = ["derive"] }serde_json = { workspace = true }tokio = { workspace = true, features = ["macros", "rt-multi-thread", "sync", "time"] }+tracing = { workspace = true }[dev-dependencies]ctor = { workspace = true }diff --git a/codex-rs/realtime-webrtc/src/client.rs b/codex-rs/realtime-webrtc/src/client.rsindex 5f5138ecff..aa56305646 100644--- a/codex-rs/realtime-webrtc/src/client.rs+++ b/codex-rs/realtime-webrtc/src/client.rs@@ -12,6 +12,7 @@ use codex_utils_pty::SpawnedProcess;use tokio::sync::oneshot;use tokio::time::timeout;+use crate::HelperExitStage;use crate::Message;use crate::encode_frame;use crate::message_reader::MessageReader;@@ -41,6 +42,7 @@ pub struct VoiceHost {process: ProcessHandle,output: MessageReader,exit: oneshot::Receiver<i32>,+ observed_exit: Option<Option<i32>>,}impl VoiceHost {@@ -73,7 +75,7 @@ impl VoiceHost {.map_err(|_| anyhow::anyhow!("voice helper input unavailable"))?;let deadline = tokio::time::Instant::now() + DEADLINE;Ok(async move {- let response = tokio::time::timeout_at(deadline, self.output.next()).await??;+ let response = tokio::time::timeout_at(deadline, self.next_response()).await??;ensure!(response == Message::AudioControlsApplied {},"unexpected voice helper response"@@ -166,6 +168,7 @@ impl VoiceHost {process: session,output: MessageReader::new(stdout_rx),exit: exit_rx,+ observed_exit: None,};host.exchange(Message::Hello {@@ -187,7 +190,11 @@ impl VoiceHost {if result.is_err() {self.process.terminate();}- let code = timeout(DEADLINE, &mut self.exit).await??;+ let code = match self.observed_exit {+ Some(Some(code)) => code,+ Some(None) => anyhow::bail!("voice helper exit status unavailable"),+ None => timeout(DEADLINE, &mut self.exit).await??,+ };result?;ensure!(code == 0, "voice helper failed during shutdown");Ok(())@@ -213,10 +220,37 @@ impl VoiceHost {.send(encode_frame(&request)?).await.map_err(|_| anyhow::anyhow!("voice helper input closed"))?;- Ok(self.output.next().await?)+ self.next_response().await}).await?}++ async fn next_response(&mut self) -> Result<Message> {+ match self.output.next().await {+ Ok(response) => Ok(response),+ Err(error) if error.kind() == std::io::ErrorKind::UnexpectedEof => {+ // The helper can exit just after closing stdout. Wait briefly for its+ // status; never log its untyped stderr, SDP, or native error text.+ let exit_code = match self.observed_exit {+ Some(status) => status,+ None => {+ match timeout(Duration::from_millis(/*millis*/ 250), &mut self.exit).await {+ Ok(status) => {+ let status = status.ok();+ self.observed_exit = Some(status);+ status+ }+ Err(_) => None,+ }+ }+ };+ let phase = exit_code.and_then(HelperExitStage::from_code);+ tracing::warn!(?phase, ?exit_code, "voice helper output closed");+ Err(error.into())+ }+ Err(error) => Err(error.into()),+ }+ }}fn child_environment(vars: impl Iterator<Item = (OsString, OsString)>) -> HashMap<String, String> {diff --git a/codex-rs/realtime-webrtc/src/client_tests.rs b/codex-rs/realtime-webrtc/src/client_tests.rsindex e0c96ab7f3..7f77d719ff 100644--- a/codex-rs/realtime-webrtc/src/client_tests.rs+++ b/codex-rs/realtime-webrtc/src/client_tests.rs@@ -50,6 +50,7 @@ async fn cancelling_initialization_terminates_the_owned_helper() -> anyhow::Resuprocess: spawned.session,output: crate::message_reader::MessageReader::new(spawned.stdout_rx),exit: unused_exit,+ observed_exit: None,};let mut initialization = Box::pin(host.initialize_runtime());std::future::poll_fn(|context| {@@ -83,6 +84,7 @@ async fn negotiation_timeout_waits_for_helper_exit_before_allowing_retry() -> anprocess: spawned.session,output: crate::message_reader::MessageReader::new(receiver),exit,+ observed_exit: None,};let answer = crate::SessionDescription::try_from("synthetic-answer".to_owned()).unwrap();let mut connection = Box::pin(host.apply_answer(answer));@@ -110,3 +112,32 @@ async fn negotiation_timeout_waits_for_helper_exit_before_allowing_retry() -> an);Ok(())}++#[cfg(unix)]+#[tokio::test]+async fn close_after_output_eof_reuses_observed_exit() -> anyhow::Result<()> {+ let spawned = codex_utils_pty::spawn_pipe_process(+ std::path::Path::new("/bin/sleep"),+ &["30".to_owned()],+ std::path::Path::new("/"),+ &child_environment(std::iter::empty()),+ /*arg0*/ &None,+ &[],+ )+ .await?;+ let (output, receiver) = tokio::sync::mpsc::channel(/*buffer*/ 1);+ drop(output);+ let (reaped, exit) = tokio::sync::oneshot::channel();+ let exit_code = crate::HelperExitStage::AudioService.code();+ reaped.send(exit_code).unwrap();+ let mut host = super::VoiceHost {+ process: spawned.session,+ output: crate::message_reader::MessageReader::new(receiver),+ exit,+ observed_exit: None,+ };+ assert!(host.next_response().await.is_err());+ assert_eq!(host.observed_exit, Some(Some(exit_code)));+ assert!(host.close().await.is_err());+ Ok(())+}diff --git a/codex-rs/realtime-webrtc/src/helper_exit.rs b/codex-rs/realtime-webrtc/src/helper_exit.rsnew file mode 100644index 0000000000..36c23f79e5--- /dev/null+++ b/codex-rs/realtime-webrtc/src/helper_exit.rs@@ -0,0 +1,55 @@+//! Fixed helper exit phases for safe diagnostics when its framed output closes.+//! The same-build handshake keeps these codes aligned between parent and helper.++#[derive(Clone, Copy, Debug)]+#[repr(i32)]+pub enum HelperExitStage {+ ControlRead = 20,+ ControlQueue = 21,+ ParentGone = 22,+ Runtime = 23,+ Transport = 24,+ OpenDevices = 25,+ AudioIngress = 26,+ AudioService = 27,+ InspectAudio = 28,+ AudioControls = 29,+ Reply = 30,+ Shutdown = 31,+ ControlSequence = 32,+ Playout = 33,+ Render = 35,+ Capture = 36,+ Device = 37,+ Send = 39,+}++impl HelperExitStage {+ pub fn code(self) -> i32 {+ self as i32+ }++ pub fn from_code(code: i32) -> Option<Self> {+ Some(match code {+ 20 => Self::ControlRead,+ 21 => Self::ControlQueue,+ 22 => Self::ParentGone,+ 23 => Self::Runtime,+ 24 => Self::Transport,+ 25 => Self::OpenDevices,+ 26 => Self::AudioIngress,+ 27 => Self::AudioService,+ 28 => Self::InspectAudio,+ 29 => Self::AudioControls,+ 30 => Self::Reply,+ 31 => Self::Shutdown,+ 32 => Self::ControlSequence,+ 33 => Self::Playout,+ 35 => Self::Render,+ 36 => Self::Capture,+ 37 => Self::Device,+ 39 => Self::Send,+ _ => return None,+ })+ }+}diff --git a/codex-rs/realtime-webrtc/src/lib.rs b/codex-rs/realtime-webrtc/src/lib.rsindex 8abfa53eb3..6ce6b1d732 100644--- a/codex-rs/realtime-webrtc/src/lib.rs+++ b/codex-rs/realtime-webrtc/src/lib.rs@@ -1,10 +1,12 @@mod client;+mod helper_exit;mod message_reader;mod protocol;mod session;pub use client::ConnectionError;pub use client::VoiceHost;+pub use helper_exit::HelperExitStage;pub use protocol::AudioControls;pub use protocol::AudioState;pub use protocol::MAX_FRAME_BYTES;diff --git a/codex-rs/realtime-webrtc/src/session.rs b/codex-rs/realtime-webrtc/src/session.rsindex b699c03e4c..a0f95b4bfe 100644--- a/codex-rs/realtime-webrtc/src/session.rs+++ b/codex-rs/realtime-webrtc/src/session.rs@@ -133,11 +133,14 @@ impl RealtimeWebrtcSession {.name("voice-session".into()).spawn(move || {let task = async {- let host = VoiceHost::connect(&package, &build_commit)- .await?- .initialize_runtime()- .await?;- let (host, sdp) = host.start_transport().await?;+ let host = report_failure(+ "connect",+ VoiceHost::connect(&package, &build_commit).await,+ )?;+ let host =+ report_failure("initialize_runtime", host.initialize_runtime().await)?;+ let (host, sdp) =+ report_failure("start_transport", host.start_transport().await)?;offer.send(sdp.into_sdp()).map_err(|_| anyhow::anyhow!("voice startup cancelled"))?;@@ -274,24 +277,25 @@ async fn run(return Ok(());}};- host = host.open_devices().await?;+ host = report_failure("open_devices", host.open_devices().await)?;let applied = startup_controls(&mut commands, controls, |initial| {host.begin_audio_controls(initial)- })?;- applied.await?;+ });+ let applied = report_failure("queue_startup_controls", applied)?;+ report_failure("apply_startup_controls", applied.await)?;connected = true;let _ = complete.send(Ok(()));}Some(Command::Controls(next)) => {if connected {- host.set_audio_controls(next).await?;+ report_failure("set_audio_controls", host.set_audio_controls(next).await)?;}}Some(Command::Answer(..)) => anyhow::bail!("voice answer already applied"),- None => return host.close().await,+ None => return report_failure("close", host.close().await),},_ = poll.tick() => {- let audio = host.inspect_audio().await?;+ let audio = report_failure("inspect_audio", host.inspect_audio().await)?;state.microphone.fetch_max(audio.microphone_peak, Ordering::Release);state.speaker.fetch_max(audio.speaker_peak, Ordering::Release);}@@ -299,6 +303,26 @@ async fn run(}}+// Keep diagnostics bounded and independent of untyped native, SDP, or device error text.+fn report_failure<T>(stage: &'static str, result: Result<T>) -> Result<T> {+ result.inspect_err(|error| {+ let kind = if error.is::<tokio::time::error::Elapsed>() {+ "timeout"+ } else if let Some(error) = error.downcast_ref::<std::io::Error>() {+ match error.kind() {+ std::io::ErrorKind::UnexpectedEof+ | std::io::ErrorKind::BrokenPipe+ | std::io::ErrorKind::ConnectionReset => "closed",+ std::io::ErrorKind::InvalidData => "protocol",+ _ => "io",+ }+ } else {+ "other"+ };+ tracing::warn!(stage, kind, "voice session operation failed");+ })+}+// Devices are still disabled. The snapshot and request enqueue share the setters' lock;// later setters remain ordered after this request without waiting for its acknowledgement.fn startup_controls<T>(diff --git a/codex-rs/tui/src/startup_orchestration.rs b/codex-rs/tui/src/startup_orchestration.rsindex 3ccc92c4f9..db0e05f63b 100644--- a/codex-rs/tui/src/startup_orchestration.rs+++ b/codex-rs/tui/src/startup_orchestration.rs@@ -562,7 +562,9 @@ pub(super) async fn run_main_inner(let log_file = log_file_opts.open(log_dir.join(TUI_LOG_FILE_NAME))?;let (non_blocking, guard) = non_blocking(log_file);let env_filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| {- EnvFilter::new("codex_core=info,codex_tui=info,codex_rmcp_client=info")+ EnvFilter::new(+ "codex_core=info,codex_tui=info,codex_rmcp_client=info,codex_realtime_webrtc=warn",+ )});let file_layer = tracing_subscriber::fmt::layer().with_writer(non_blocking)diff --git a/codex-rs/voice-host/src/audio_track.rs b/codex-rs/voice-host/src/audio_track.rsindex 1d3de94890..1525465e7d 100644--- a/codex-rs/voice-host/src/audio_track.rs+++ b/codex-rs/voice-host/src/audio_track.rs@@ -1,4 +1,5 @@-//! A single Opus RTP track. Elapsed mute time advances its clock without inventing packet loss.+//! A single Opus RTP track. Muted capture sends generated silence to keep the peer alive;+//! elapsed time still advances its clock without inventing packet loss.use std::sync::Arc;use std::time::Duration;@@ -26,6 +27,7 @@ pub(crate) struct AudioTrack {pub(crate) track: Arc<TrackLocalStaticSample>,ssrc: u32,end: Option<Instant>,+ silence: Option<Vec<u8>>,}impl AudioTrack {@@ -70,10 +72,47 @@ impl AudioTrack {track: Arc::new(track),ssrc,end: None,+ silence: None,},))}+ /// Send only synthetic silence while muted, never a device or processing buffer.+ /// Reuse the encoded packet and pace it by the same RTP clock as live capture.+ #[cfg(any(+ target_os = "macos",+ all(target_os = "linux", target_env = "gnu"),+ all(windows, target_env = "msvc")+ ))]+ pub(crate) async fn send_muted_silence(&mut self, at: Instant) -> Result<(), &'static str> {+ if self.end.is_some_and(|end| at < end) {+ return Ok(());+ }+ if self.silence.is_none() {+ let mut encoder = opus::Encoder::new(+ /*sample_rate*/ 48_000,+ opus::Channels::Mono,+ opus::Application::Voip,+ )+ .map_err(|_| "failed to create voice silence encoder")?;+ let mut data = vec![0; 1275];+ let len = encoder+ .encode_float(&[0.0; 960], &mut data)+ .map_err(|_| "failed to encode voice silence")?;+ data.truncate(len);+ self.silence = Some(data);+ }+ self.send(EncodedAudio {+ data: self+ .silence+ .as_ref()+ .ok_or("voice silence unavailable")?+ .clone(),+ at,+ })+ .await+ }+pub(crate) async fn send(&mut self, frame: EncodedAudio) -> Result<(), &'static str> {tokio::time::timeout(SEND_TIMEOUT, async {let mut gap = selfdiff --git a/codex-rs/voice-host/src/capture_worker.rs b/codex-rs/voice-host/src/capture_worker.rsindex c97ab53f33..dc6f8ef47d 100644--- a/codex-rs/voice-host/src/capture_worker.rs+++ b/codex-rs/voice-host/src/capture_worker.rs@@ -1,5 +1,6 @@//! Pumps bounded device queues through capture processing and the outgoing audio track.//! Devices retain real stream ownership; all queued and partial capture obey mute generations.+//! Muted sessions send generated Opus silence independently of device capture.//! Retain at most one encoded batch and return between packets so controls can take priority.use std::collections::VecDeque;@@ -11,6 +12,7 @@ use std::time::Instant;use super::MAX_CAPTURE_AGE;use super::buffers::Buffers;use super::processing;+use crate::service_failure::ServiceFailure;pub(super) struct CaptureWorker {pub(super) buffers: Arc<Buffers>,@@ -43,36 +45,90 @@ impl CaptureWorker {// Some backends start callbacks during stream construction, before open returns.self.buffers.serviced.store(true, Ordering::Release);let generation = self.buffers.microphone.load(Ordering::Acquire);+ if self.buffers.render_dropped.swap(false, Ordering::AcqRel) {+ self.processor.reset_render();+ for _ in 0..self.buffers.rendered.capacity() {+ if self.buffers.rendered.pop().is_none() {+ break;+ }+ }+ }+ if self.buffers.capture_dropped.swap(false, Ordering::AcqRel) {+ self.pending.clear();+ self.processor.reset().map_err(io::Error::other)?;+ for _ in 0..self.buffers.capture.capacity() {+ if self.buffers.capture.pop().is_none() {+ break;+ }+ }+ }+ let mut render_stale = false;for _ in 0..self.buffers.rendered.capacity() {let Some(frame) = self.buffers.rendered.pop() else {break;};if frame.at.elapsed() > MAX_CAPTURE_AGE {- return Err(io::Error::other("audio rendering fell behind"));+ if !render_stale {+ self.processor.reset_render();+ render_stale = true;+ }+ continue;}- self.processor.render(&frame).map_err(io::Error::other)?;+ render_stale = false;+ self.processor+ .render(&frame)+ .map_err(|_| io::Error::other(ServiceFailure::Render))?;}if self.pending.is_empty() {+ let mut capture_stale = false;for _ in 0..self.buffers.capture.capacity() {let Some(frame) = self.buffers.capture.pop() else {break;};if generation.is_multiple_of(2) && frame.generation == generation {+ if now().saturating_duration_since(frame.at) > processing::MAX_PROCESSING_DELAY+ {+ self.pending.clear();+ if !capture_stale {+ self.processor.reset().map_err(io::Error::other)?;+ capture_stale = true;+ }+ continue;+ }+ capture_stale = false;let frames = self.processor.capture(&frame, &now);- self.pending.extend(frames.map_err(io::Error::other)?);+ match frames {+ Ok(frames) => self.pending.extend(frames),+ Err(processing::PROCESSING_LATE) => {+ self.pending.clear();+ self.processor.reset().map_err(io::Error::other)?;+ }+ Err(_) => return Err(io::Error::other(ServiceFailure::Capture)),+ }}}}if self.buffers.failed.load(Ordering::Acquire) {- return Err(io::Error::other("audio device failed"));+ return Err(io::Error::other(ServiceFailure::Device));}let Some(frame) = self.pending.pop_front() else {+ if !generation.is_multiple_of(2) {+ audio+ .send_muted_silence(now())+ .await+ .map_err(|_| io::Error::other(ServiceFailure::Send))?;+ }return Ok(0);};if now().saturating_duration_since(frame.at) > processing::MAX_PROCESSING_DELAY {- return Err(io::Error::other("voice processing fell behind"));+ self.pending.clear();+ self.processor.reset().map_err(io::Error::other)?;+ return Ok(0);}- audio.send(frame).await.map_err(io::Error::other)?;+ audio+ .send(frame)+ .await+ .map_err(|_| io::Error::other(ServiceFailure::Send))?;Ok(1)}}diff --git a/codex-rs/voice-host/src/capture_worker_tests.rs b/codex-rs/voice-host/src/capture_worker_tests.rsindex ef18870307..17e739b0bf 100644--- a/codex-rs/voice-host/src/capture_worker_tests.rs+++ b/codex-rs/voice-host/src/capture_worker_tests.rs@@ -267,12 +267,32 @@ async fn capture_reaches_remote_rtp_and_mute_discards_queued_and_partial_audio()worker.set_controls(muted()).unwrap();worker.set_controls(unmuted()).unwrap();let old_generation = worker.buffers.microphone.load(Ordering::Acquire);- now = Instant::now();+ now = Instant::now().max(start + Duration::from_millis(/*millis*/ 100));enqueue(&worker, /*samples*/ 720, old_generation, now);assert_eq!(worker.service(&mut local.audio, || now).await.unwrap(), 0);enqueue(&worker, /*samples*/ 2_880, old_generation, now);worker.set_controls(muted()).unwrap();assert_eq!(worker.service(&mut local.audio, || now).await.unwrap(), 0);+ let silence = received.recv().await.unwrap();+ let mut decoder = opus::Decoder::new(/*sample_rate*/ 48_000, opus::Channels::Mono).unwrap();+ let mut decoded = [1.0; 960];+ assert_eq!(+ decoder+ .decode_float(&silence.payload, &mut decoded, /*fec*/ false)+ .unwrap(),+ 960+ );+ assert!(decoded.iter().all(|sample| sample.abs() < 0.0001));+ assert_eq!(+ silence.header.sequence_number,+ before+ .last()+ .unwrap()+ .header+ .sequence_number+ .wrapping_add(1)+ );+ assert!(received.try_recv().is_err());// A callback that began before mute may enqueue its old generation only// after unmute; a newly started callback may still carry an old device@@ -304,12 +324,7 @@ async fn capture_reaches_remote_rtp_and_mute_discards_queued_and_partial_audio());assert_eq!(after[0].header.sequence_number,- before- .last()- .unwrap()- .header- .sequence_number- .wrapping_add(1)+ silence.header.sequence_number.wrapping_add(1));assert!(received.try_recv().is_err());@@ -337,22 +352,77 @@ async fn capture_reaches_remote_rtp_and_mute_discards_queued_and_partial_audio()assert_eq!(worker.service(&mut local.audio, || now).await.unwrap(), 0);assert!(received.try_recv().is_err());- // Waiting behind a slow send cannot extend the capture freshness limit.+ // Waiting behind a slow send discards stale audio without ending the session.let generation = worker.buffers.microphone.load(Ordering::Acquire);now = Instant::now() + Duration::from_secs(/*secs*/ 1);enqueue(&worker, /*samples*/ 2_880, generation, now);assert_eq!(worker.service(&mut local.audio, || now).await.unwrap(), 1);received.recv().await.unwrap();let stale = now + Duration::from_secs(/*secs*/ 1);+ assert_eq!(worker.service(&mut local.audio, || stale).await.unwrap(), 0);+ assert!(received.try_recv().is_err());+ now = stale + Duration::from_millis(/*millis*/ 10);+ enqueue(&worker, /*samples*/ 2_880, generation, now);+ let resumed = receive_capture(&mut worker, &mut local, &mut received, now).await;+ assert!(!resumed.is_empty());++ // A saturated callback queue also retires partial processing state, then+ // admits new capture once the worker catches up.+ enqueue(&worker, /*samples*/ 2_880, generation, now);+ worker+ .buffers+ .capture_dropped+ .store(true, Ordering::Release);+ assert_eq!(worker.service(&mut local.audio, || now).await.unwrap(), 0);+ assert!(received.try_recv().is_err());+ now += Duration::from_millis(/*millis*/ 70);+ enqueue(&worker, /*samples*/ 2_880, generation, now);+ assert!(+ !receive_capture(&mut worker, &mut local, &mut received, now)+ .await+ .is_empty()+ );++ // A delayed worker must discard a full render-reference queue before+ // it feeds stale speaker audio back into echo cancellation.+ let mut output = [0.0_f32; BLOCK];+ let mut output_state = super::super::OutputState::default();+ for _ in 0..=worker.buffers.rendered.capacity() {+ super::super::render_output(+ &mut output,+ /*channels*/ 1,+ /*rate*/ 44_100.0,+ now,+ &worker.buffers,+ &mut output_state,+ );+ }assert_eq!(- worker- .service(&mut local.audio, || stale)+ worker.buffers.rendered.len(),+ worker.buffers.rendered.capacity()+ );+ assert!(worker.buffers.render_dropped.load(Ordering::Acquire));+ assert_eq!(worker.service(&mut local.audio, || now).await.unwrap(), 0);+ assert!(worker.buffers.rendered.is_empty());+ assert!(!worker.buffers.render_dropped.load(Ordering::Acquire));+ assert!(!worker.buffers.failed.load(Ordering::Acquire));++ now += Duration::from_millis(/*millis*/ 70);+ super::super::render_output(+ &mut output,+ /*channels*/ 1,+ /*rate*/ 44_100.0,+ now,+ &worker.buffers,+ &mut output_state,+ );+ enqueue(&worker, /*samples*/ 2_880, generation, now);+ assert!(+ !receive_capture(&mut worker, &mut local, &mut received, now).await- .unwrap_err()- .to_string(),- "voice processing fell behind"+ .is_empty());- assert!(received.try_recv().is_err());+ assert!(worker.buffers.rendered.is_empty());local.close().await.unwrap();remote.close().await.unwrap();})diff --git a/codex-rs/voice-host/src/device_buffers.rs b/codex-rs/voice-host/src/device_buffers.rsindex 0609d9dbae..b38f13fe2e 100644--- a/codex-rs/voice-host/src/device_buffers.rs+++ b/codex-rs/voice-host/src/device_buffers.rs@@ -98,6 +98,8 @@ pub(super) struct Buffers {pub(super) speaker_failure_gate: Mutex<()>,pub(super) serviced: AtomicBool,pub(super) failed: AtomicBool,+ pub(super) capture_dropped: AtomicBool,+ pub(super) render_dropped: AtomicBool,pub(super) microphone_peak: AtomicU16,pub(super) speaker_peak: AtomicU16,pub(super) queued: AtomicU32,@@ -136,6 +138,8 @@ impl Buffers {speaker_failure_gate: Mutex::new(()),serviced: AtomicBool::new(false),failed: AtomicBool::new(false),+ capture_dropped: AtomicBool::new(false),+ render_dropped: AtomicBool::new(false),microphone_peak: AtomicU16::new(/*v*/ 0),speaker_peak: AtomicU16::new(/*v*/ 0),queued: AtomicU32::new(/*v*/ 0),diff --git a/codex-rs/voice-host/src/devices.rs b/codex-rs/voice-host/src/devices.rsindex 5fcaab5232..1f1d58f1f1 100644--- a/codex-rs/voice-host/src/devices.rs+++ b/codex-rs/voice-host/src/devices.rs@@ -1,5 +1,5 @@//! Owns local device streams on the helper worker. Callbacks allocate no buffers and take no locks.-//! Small callbacks share full queue slots; processing lag still fails the session closed.+//! Small callbacks share full queue slots; overload discards stale media and resumes fresh audio.//! Capture and actual rendered output carry device timing.//! References start with worker service; unmute rejects earlier device capture buffers.@@ -207,7 +207,9 @@ impl Devices {audio: &mut crate::audio_track::AudioTrack,) -> io::Result<usize> {if let Some(playout) = &self.playout {- playout.check().map_err(io::Error::other)?;+ playout+ .check()+ .map_err(|_| io::Error::other(crate::service_failure::ServiceFailure::Playout))?;}self.worker.service(audio, Instant::now).await}@@ -338,7 +340,8 @@ whererecord_peak(&buffers.microphone_peak, *output);}if !capture.push(frame, rate, &buffers.capture) {- buffers.failed.store(true, Ordering::Release);+ capture.reset();+ buffers.capture_dropped.store(true, Ordering::Release);return;}}@@ -429,7 +432,8 @@ fn render_output<T>(record_peak(&buffers.speaker_peak, *sample);}if !output.reference.push(reference, rate, &buffers.rendered) {- buffers.failed.store(true, Ordering::Release);+ output.reference.reset();+ buffers.render_dropped.store(true, Ordering::Release);}}if let Some(end) = delivered_until {diff --git a/codex-rs/voice-host/src/devices_tests.rs b/codex-rs/voice-host/src/devices_tests.rsindex f313fe61f1..2bfcffee77 100644--- a/codex-rs/voice-host/src/devices_tests.rs+++ b/codex-rs/voice-host/src/devices_tests.rs@@ -2,7 +2,7 @@ use super::*;use pretty_assertions::assert_eq;#[test]-fn startup_output_waits_for_service_but_later_overflow_still_fails() {+fn startup_output_waits_for_service_and_later_overflow_discards_reference() {let buffers = Buffers::new(/*input_rate*/ 48_000, /*output_rate*/ 384_000);let mut state = OutputState::default();let mut output = [1.0_f32; BLOCK * 2];@@ -65,8 +65,9 @@ fn startup_output_waits_for_service_but_later_overflow_still_fails() {&buffers,&mut state,);- assert!(buffers.failed.load(Ordering::Acquire));- assert!(buffers.take_state().is_err());+ assert!(!buffers.failed.load(Ordering::Acquire));+ assert!(buffers.render_dropped.load(Ordering::Acquire));+ assert!(buffers.take_state().is_ok());}#[test]@@ -332,7 +333,8 @@ fn suppressed_output_does_not_fail_when_reference_queue_is_full() {&buffers,&mut state,);- assert!(buffers.failed.load(Ordering::Acquire));+ assert!(!buffers.failed.load(Ordering::Acquire));+ assert!(buffers.render_dropped.load(Ordering::Acquire));}#[test]diff --git a/codex-rs/voice-host/src/incoming.rs b/codex-rs/voice-host/src/incoming.rsindex b230d0eda5..89c46d1fff 100644--- a/codex-rs/voice-host/src/incoming.rs+++ b/codex-rs/voice-host/src/incoming.rs@@ -1,5 +1,6 @@//! Takes incoming Opus RTP before the upstream track queue, preserving network arrival time.//! Packet and byte permits follow owned data through the consumer, not merely through this queue.+//! Stale or saturated media is discarded so a slow worker can resume with fresh speaker audio.use std::sync::Arc;use std::sync::atomic::AtomicBool;@@ -106,7 +107,7 @@ impl Incoming {};if epoch.is_multiple_of(2) && packet.epoch == epoch {if packet.at.elapsed() > Duration::from_secs(/*secs*/ 1) {- return Err("incoming audio fell behind");+ continue;}return Ok(Some(packet));}@@ -127,38 +128,35 @@ impl Ingress {return Ok(());}let size = packet.marshal_size();- let accepted = (|| {- if size > PACKET_BYTES- || packet.header.payload_type != crate::audio_track::OPUS_PAYLOAD_TYPE- || self.stream.is_some_and(|ssrc| ssrc != packet.header.ssrc)- {- return None;- }- self.stream = Some(packet.header.ssrc);- let packet_permit = self.state.packets.clone().try_acquire_owned().ok()?;- let bytes_permit = self- .state- .bytes- .clone()- .try_acquire_many_owned(size as u32)- .ok()?;- let mut data = vec![0; size];- if packet.marshal_to(&mut data).ok()? != size {- return None;- }- self.sender- .try_send(ReceivedRtp {- data,- at: message.now,- epoch,- _packet: packet_permit,- _bytes: bytes_permit,- })- .ok()- })();- if accepted.is_none() {+ if size > PACKET_BYTES+ || packet.header.payload_type != crate::audio_track::OPUS_PAYLOAD_TYPE+ || self.stream.is_some_and(|ssrc| ssrc != packet.header.ssrc)+ {self.state.failed.store(true, Ordering::Release);+ return Ok(());}+ self.stream = Some(packet.header.ssrc);+ if message.now.elapsed() > Duration::from_secs(/*secs*/ 1) {+ return Ok(());+ }+ let Ok(packet_permit) = self.state.packets.clone().try_acquire_owned() else {+ return Ok(());+ };+ let Ok(bytes_permit) = self.state.bytes.clone().try_acquire_many_owned(size as u32) else {+ return Ok(());+ };+ let mut data = vec![0; size];+ if packet.marshal_to(&mut data).ok() != Some(size) {+ self.state.failed.store(true, Ordering::Release);+ return Ok(());+ }+ let _ = self.sender.try_send(ReceivedRtp {+ data,+ at: message.now,+ epoch,+ _packet: packet_permit,+ _bytes: bytes_permit,+ });// Consume here: no second copy is retained by the upstream track-event queue.Ok(())}diff --git a/codex-rs/voice-host/src/incoming_tests.rs b/codex-rs/voice-host/src/incoming_tests.rsindex bc32d83845..70d6048e21 100644--- a/codex-rs/voice-host/src/incoming_tests.rs+++ b/codex-rs/voice-host/src/incoming_tests.rs@@ -41,7 +41,7 @@ fn arrival_and_rtp_bytes_survive_the_adapter_without_an_upstream_copy() {}#[test]-fn consumed_packets_keep_their_budget_until_the_last_owner_drops_them() {+fn exhausted_packet_or_byte_budget_drops_audio_then_accepts_fresh_packets() {for size in [32, PACKET_BYTES] {let (mut incoming, mut ingress) = pair();let count = PACKETS.min(BYTES / size);@@ -54,8 +54,11 @@ fn consumed_packets_keep_their_budget_until_the_last_owner_drops_them() {ingress.handle_read(packet(size, Instant::now())).unwrap();held.push(incoming.take().unwrap().unwrap());ingress.handle_read(packet(size, Instant::now())).unwrap();- assert_eq!(incoming.take().err(), Some("incoming audio failed"));+ assert!(incoming.take().unwrap().is_none());+ assert!(!incoming.state.failed.load(Ordering::Acquire));drop(held);+ ingress.handle_read(packet(size, Instant::now())).unwrap();+ assert!(incoming.take().unwrap().is_some());assert_eq!((incoming.state.packets.available_permits(),@@ -66,6 +69,22 @@ fn consumed_packets_keep_their_budget_until_the_last_owner_drops_them() {}}+#[test]+fn old_speaker_packets_are_discarded_without_poisoning_fresh_audio() {+ let (mut incoming, mut ingress) = pair();+ ingress+ .handle_read(packet(/*size*/ 32, Instant::now()))+ .unwrap();+ let mut queued = incoming.receiver.try_recv().unwrap();+ queued.at -= Duration::from_secs(/*secs*/ 2);+ ingress.sender.try_send(queued).unwrap();+ assert!(incoming.take().unwrap().is_none());+ ingress+ .handle_read(packet(/*size*/ 32, Instant::now()))+ .unwrap();+ assert!(incoming.take().unwrap().is_some());+}+#[test]fn suppression_discards_queued_and_in_flight_old_epochs() {let (mut incoming, mut ingress) = pair();diff --git a/codex-rs/voice-host/src/main.rs b/codex-rs/voice-host/src/main.rsindex 20c252eb21..709521aa0e 100644--- a/codex-rs/voice-host/src/main.rs+++ b/codex-rs/voice-host/src/main.rs@@ -22,17 +22,21 @@ mod audio_track;mod devices;mod incoming;mod runtime;+mod service_failure;mod transport;mod transport_runtime;+use std::cell::Cell;use std::io;use std::io::Write;use std::sync::mpsc;use std::time::Duration;+use codex_realtime_webrtc::HelperExitStage;use codex_realtime_webrtc::Message;use codex_realtime_webrtc::encode_frame;use codex_realtime_webrtc::read_message;+use service_failure::ServiceFailure;const DEVICE_SERVICE_INTERVAL: Duration = Duration::from_millis(/*millis*/ 5);@@ -47,14 +51,16 @@ fn main() {match (args.next(), args.next()) {(Some(arg), None) if arg == "--build-commit" => println!("{BUILD_COMMIT}"),(None, None) => {- if run(|executor| {+ let phase = Cell::new(HelperExitStage::ControlSequence);+ if run(&phase, |executor| {+ phase.set(HelperExitStage::Transport);executor.block_on(transport::Transport::new()).map_err(io::Error::other)}).is_err(){- std::process::exit(/*code*/ 1);+ std::process::exit(phase.get().code());}}_ => std::process::exit(/*code*/ 2),@@ -62,6 +68,7 @@ fn main() {}fn run(+ phase: &Cell<HelperExitStage>,start_transport: impl Fn(&tokio::runtime::Runtime) -> io::Result<transport::Transport>,) -> io::Result<()> {let (sender, receiver) = mpsc::sync_channel(/*bound*/ 1);@@ -73,17 +80,17 @@ fn run(match read_message(&mut input) {Ok(Some(message)) => {if sender.try_send(message).is_err() {- std::process::exit(/*code*/ 1);+ std::process::exit(HelperExitStage::ControlQueue.code());}}Ok(None) => break,- Err(_) => std::process::exit(/*code*/ 1),+ Err(_) => std::process::exit(HelperExitStage::ControlRead.code()),}}drop(sender);// Independent of the main worker or a blocked stdout write, including after parent death.std::thread::sleep(Duration::from_secs(/*secs*/ 2));- std::process::exit(/*code*/ 1);+ std::process::exit(HelperExitStage::ParentGone.code());})?;let Ok(hello) = receiver.recv() else {return Ok(());@@ -96,6 +103,7 @@ fn run({return Err(io::Error::other("incompatible voice helper"));}+ phase.set(HelperExitStage::Reply);let mut output = io::stdout().lock();output.write_all(&encode_frame(&Message::Ready {})?)?;output.flush()?;@@ -105,6 +113,7 @@ fn run(let mut answered = false;let mut devices: Option<devices::Devices> = None;loop {+ phase.set(HelperExitStage::ControlSequence);let message = if let Some(devices) = &mut devices {let peer = transport.as_mut()@@ -113,12 +122,29 @@ fn run(// Bound one pass to the ingress queue capacity so new media// cannot indefinitely postpone a waiting privacy control.for _ in 0..64 {+ phase.set(HelperExitStage::AudioIngress);let Some(packet) = peer.incoming.take().map_err(io::Error::other)? else {break;};+ phase.set(HelperExitStage::Playout);devices.receive(packet)?;}- executor.block_on(devices.service(&mut peer.audio))?;+ phase.set(HelperExitStage::AudioService);+ if let Err(error) = executor.block_on(devices.service(&mut peer.audio)) {+ if let Some(failure) = error+ .get_ref()+ .and_then(|error| error.downcast_ref::<ServiceFailure>())+ {+ phase.set(match failure {+ ServiceFailure::Playout => HelperExitStage::Playout,+ ServiceFailure::Render => HelperExitStage::Render,+ ServiceFailure::Capture => HelperExitStage::Capture,+ ServiceFailure::Device => HelperExitStage::Device,+ ServiceFailure::Send => HelperExitStage::Send,+ });+ }+ return Err(error);+ }Ok(())})?.ok_or(mpsc::RecvTimeoutError::Disconnected)@@ -129,6 +155,7 @@ fn run(};let reply = match message {Ok(Message::InspectAudio {}) => {+ phase.set(HelperExitStage::InspectAudio);if answered && transport.as_ref().is_none_or(|peer| !*peer.ready.borrow()) {return Err(io::Error::other("voice connection closed"));}@@ -136,15 +163,18 @@ fn run(state: devices.as_ref().map(devices::Devices::take_state)- .transpose()?+ .transpose()+ .inspect_err(|_| phase.set(HelperExitStage::Device))?.unwrap_or_default(),}}Ok(Message::OpenDevices {}) if devices.is_none() && runtime.is_some() && answered => {+ phase.set(HelperExitStage::OpenDevices);devices = Some(devices::Devices::open()?);Message::DevicesOpened {}}Ok(Message::SetAudioControls { controls }) => {+ phase.set(HelperExitStage::AudioControls);let peer = transport.as_ref().ok_or_else(|| io::Error::other("voice peer not started"))?;@@ -165,6 +195,7 @@ fn run(Message::AudioControlsApplied {}}Ok(Message::StartTransport {}) if transport.is_none() => {+ phase.set(HelperExitStage::Transport);let peer = start_transport(&executor)?;let sdp = executor.block_on(peer.offer()).map_err(io::Error::other)?;transport = Some(peer);@@ -173,6 +204,7 @@ fn run(}}Ok(Message::ApplyAnswer { sdp }) if !answered => {+ phase.set(HelperExitStage::Transport);let Some(peer) = transport.as_ref() else {return Err(io::Error::other("voice transport not started"));};@@ -189,6 +221,7 @@ fn run(Message::TransportReady {}}Ok(Message::InitializeRuntime {}) => {+ phase.set(HelperExitStage::Runtime);if runtime.is_some() {return Err(io::Error::other("runtime already initialized"));}@@ -196,6 +229,7 @@ fn run(Message::RuntimeReady {}}Ok(Message::Close {}) => {+ phase.set(HelperExitStage::Shutdown);let _ = devices.take();if let Some(mut peer) = transport.take() {executor.block_on(peer.close()).map_err(io::Error::other)?;@@ -221,6 +255,7 @@ fn run(| Message::Closed {},) => return Err(io::Error::other("invalid voice control sequence")),};+ phase.set(HelperExitStage::Reply);output.write_all(&encode_frame(&reply)?)?;output.flush()?;}diff --git a/codex-rs/voice-host/src/main_tests.rs b/codex-rs/voice-host/src/main_tests.rsindex eeeb797b34..8dca96433c 100644--- a/codex-rs/voice-host/src/main_tests.rs+++ b/codex-rs/voice-host/src/main_tests.rs@@ -60,7 +60,7 @@ fn blocked_startup_fixture() {if std::env::var_os(CHILD_ENV).as_deref() != Some(std::ffi::OsStr::new("1")) {return;}- run(|_| {+ run(&Cell::new(HelperExitStage::Transport), |_| {io::stderr().write_all(STARTUP_BLOCKED)?;io::stderr().flush()?;loop {@@ -121,7 +121,10 @@ async fn parent_pipe_loss_terminates_helper_with_blocked_transport_startup() {assert!(child.try_wait().unwrap().is_none());drop(input);let output = child.wait_with_output().await.unwrap();- assert_eq!(output.status.code(), Some(1));+ assert_eq!(+ output.status.code(),+ Some(HelperExitStage::ParentGone.code())+ );assert_eq!((output.stdout, output.stderr), (vec![], vec![]));}).awaitdiff --git a/codex-rs/voice-host/src/playback.rs b/codex-rs/voice-host/src/playback.rsindex 8779c48bff..282f32aa5d 100644--- a/codex-rs/voice-host/src/playback.rs+++ b/codex-rs/voice-host/src/playback.rs@@ -102,7 +102,9 @@ impl PlaybackWriter {return Ok(bytes);}if Instant::now() >= deadline {- return Err("speaker fell behind");+ // The sink must consume this chunk to catch up with the device clock.+ // Dropping it keeps the bounded queue and current epoch intact.+ return Ok(frame.len * 4);}std::thread::park_timeout(Duration::from_millis(/*millis*/ 1));}diff --git a/codex-rs/voice-host/src/playback_tests.rs b/codex-rs/voice-host/src/playback_tests.rsindex 18d2796ee5..a3a45d7db8 100644--- a/codex-rs/voice-host/src/playback_tests.rs+++ b/codex-rs/voice-host/src/playback_tests.rs@@ -123,16 +123,19 @@ fn delay_contention_preserves_the_last_snapshot_without_failing_the_device() {}#[test]-fn invalid_samples_and_stalled_consumption_fail_without_growing_the_queue() {+fn invalid_samples_fail_but_stalled_consumption_drops_and_resumes() {let (buffers, _, writer) = active(/*rate*/ 8000);assert_eq!(writer.write(&f32::NAN.to_le_bytes()),Err("invalid speaker sample"));writer.write(&vec![0; BLOCK * 4]).unwrap();- assert_eq!(- writer.write(&vec![0; BLOCK * 4]),- Err("speaker fell behind")- );+ assert_eq!(writer.write(&vec![0; BLOCK * 4]), Ok(BLOCK * 4));assert_eq!(buffers.queued.load(Ordering::Acquire), BLOCK as u32);+ let mut playback = Playback::default();+ for _ in 0..BLOCK {+ assert!(playback.next(&buffers).is_some());+ }+ assert_eq!(writer.write(&vec![0; BLOCK * 4]), Ok(BLOCK * 4));+ assert!(!buffers.failed.load(Ordering::Acquire));}diff --git a/codex-rs/voice-host/src/processing.rs b/codex-rs/voice-host/src/processing.rsindex 4ba26cfb69..b05dd19365 100644--- a/codex-rs/voice-host/src/processing.rs+++ b/codex-rs/voice-host/src/processing.rs@@ -21,6 +21,7 @@ use crate::audio_track::EncodedAudio;type Result<T> = std::result::Result<T, &'static str>;const BLOCK: usize = 480;pub(super) const MAX_PROCESSING_DELAY: Duration = Duration::from_millis(/*millis*/ 500);+pub(super) const PROCESSING_LATE: &str = "voice processing fell behind";struct Converter {resampler: rubato::Async<f32>,@@ -220,11 +221,11 @@ impl Processor {while let Some((at, input)) = self.capture.next() {let capture_age = now().saturating_duration_since(at);if capture_age > MAX_PROCESSING_DELAY {- return Err("voice processing fell behind");+ return Err(PROCESSING_LATE);}let delay = (capture_age.as_millis() as i64 + self.render_delay).max(0);if delay > MAX_PROCESSING_DELAY.as_millis() as i64 {- return Err("voice processing fell behind");+ return Err(PROCESSING_LATE);}self.apm.set_stream_delay_ms(delay as i32)diff --git a/codex-rs/voice-host/src/service_failure.rs b/codex-rs/voice-host/src/service_failure.rsnew file mode 100644index 0000000000..e42b23360e--- /dev/null+++ b/codex-rs/voice-host/src/service_failure.rs@@ -0,0 +1,32 @@+//! Fixed local media service failures for diagnostics without native error text.++#[derive(Debug)]+#[cfg_attr(+ not(any(+ target_os = "macos",+ all(target_os = "linux", target_env = "gnu"),+ all(windows, target_env = "msvc")+ )),+ allow(dead_code)+)]+pub(crate) enum ServiceFailure {+ Playout,+ Render,+ Capture,+ Device,+ Send,+}++impl std::fmt::Display for ServiceFailure {+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ f.write_str(match self {+ Self::Playout => "audio playout failed",+ Self::Render => "audio rendering failed",+ Self::Capture => "audio capture processing failed",+ Self::Device => "audio device failed",+ Self::Send => "voice audio send failed",+ })+ }+}++impl std::error::Error for ServiceFailure {}