Skip to content

armgit / codex

Public Imported from GitHub
0

Lightweight coding agent that runs in your terminal

Rust

Keep voice sessions alive through mute and audio backlog (#44671)

Benjamin Carlsson28f43b0417ab
diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock
index 39c1d3accb..572ffcd05b 100644
--- a/codex-rs/Cargo.lock
+++ b/codex-rs/Cargo.lock
@@ -4309,6 +4309,7 @@ dependencies = [
"serde",
"serde_json",
"tokio",
+ "tracing",
]
[[package]]
diff --git a/codex-rs/realtime-webrtc/Cargo.toml b/codex-rs/realtime-webrtc/Cargo.toml
index 9042544395..2d1b3b422f 100644
--- a/codex-rs/realtime-webrtc/Cargo.toml
+++ b/codex-rs/realtime-webrtc/Cargo.toml
@@ -19,6 +19,7 @@ futures = { workspace = true, features = ["std"] }
serde = { workspace = true, features = ["derive"] }
serde_json = { workspace = true }
tokio = { workspace = true, features = ["macros", "rt-multi-thread", "sync", "time"] }
+tracing = { workspace = true }
[dev-dependencies]
ctor = { workspace = true }
diff --git a/codex-rs/realtime-webrtc/src/client.rs b/codex-rs/realtime-webrtc/src/client.rs
index 5f5138ecff..aa56305646 100644
--- a/codex-rs/realtime-webrtc/src/client.rs
+++ b/codex-rs/realtime-webrtc/src/client.rs
@@ -12,6 +12,7 @@ use codex_utils_pty::SpawnedProcess;
use tokio::sync::oneshot;
use tokio::time::timeout;
+use crate::HelperExitStage;
use crate::Message;
use crate::encode_frame;
use crate::message_reader::MessageReader;
@@ -41,6 +42,7 @@ pub struct VoiceHost {
process: ProcessHandle,
output: MessageReader,
exit: oneshot::Receiver<i32>,
+ observed_exit: Option<Option<i32>>,
}
impl VoiceHost {
@@ -73,7 +75,7 @@ impl VoiceHost {
.map_err(|_| anyhow::anyhow!("voice helper input unavailable"))?;
let deadline = tokio::time::Instant::now() + DEADLINE;
Ok(async move {
- let response = tokio::time::timeout_at(deadline, self.output.next()).await??;
+ let response = tokio::time::timeout_at(deadline, self.next_response()).await??;
ensure!(
response == Message::AudioControlsApplied {},
"unexpected voice helper response"
@@ -166,6 +168,7 @@ impl VoiceHost {
process: session,
output: MessageReader::new(stdout_rx),
exit: exit_rx,
+ observed_exit: None,
};
host.exchange(
Message::Hello {
@@ -187,7 +190,11 @@ impl VoiceHost {
if result.is_err() {
self.process.terminate();
}
- let code = timeout(DEADLINE, &mut self.exit).await??;
+ let code = match self.observed_exit {
+ Some(Some(code)) => code,
+ Some(None) => anyhow::bail!("voice helper exit status unavailable"),
+ None => timeout(DEADLINE, &mut self.exit).await??,
+ };
result?;
ensure!(code == 0, "voice helper failed during shutdown");
Ok(())
@@ -213,10 +220,37 @@ impl VoiceHost {
.send(encode_frame(&request)?)
.await
.map_err(|_| anyhow::anyhow!("voice helper input closed"))?;
- Ok(self.output.next().await?)
+ self.next_response().await
})
.await?
}
+
+ async fn next_response(&mut self) -> Result<Message> {
+ match self.output.next().await {
+ Ok(response) => Ok(response),
+ Err(error) if error.kind() == std::io::ErrorKind::UnexpectedEof => {
+ // The helper can exit just after closing stdout. Wait briefly for its
+ // status; never log its untyped stderr, SDP, or native error text.
+ let exit_code = match self.observed_exit {
+ Some(status) => status,
+ None => {
+ match timeout(Duration::from_millis(/*millis*/ 250), &mut self.exit).await {
+ Ok(status) => {
+ let status = status.ok();
+ self.observed_exit = Some(status);
+ status
+ }
+ Err(_) => None,
+ }
+ }
+ };
+ let phase = exit_code.and_then(HelperExitStage::from_code);
+ tracing::warn!(?phase, ?exit_code, "voice helper output closed");
+ Err(error.into())
+ }
+ Err(error) => Err(error.into()),
+ }
+ }
}
fn child_environment(vars: impl Iterator<Item = (OsString, OsString)>) -> HashMap<String, String> {
diff --git a/codex-rs/realtime-webrtc/src/client_tests.rs b/codex-rs/realtime-webrtc/src/client_tests.rs
index e0c96ab7f3..7f77d719ff 100644
--- a/codex-rs/realtime-webrtc/src/client_tests.rs
+++ b/codex-rs/realtime-webrtc/src/client_tests.rs
@@ -50,6 +50,7 @@ async fn cancelling_initialization_terminates_the_owned_helper() -> anyhow::Resu
process: spawned.session,
output: crate::message_reader::MessageReader::new(spawned.stdout_rx),
exit: unused_exit,
+ observed_exit: None,
};
let mut initialization = Box::pin(host.initialize_runtime());
std::future::poll_fn(|context| {
@@ -83,6 +84,7 @@ async fn negotiation_timeout_waits_for_helper_exit_before_allowing_retry() -> an
process: spawned.session,
output: crate::message_reader::MessageReader::new(receiver),
exit,
+ observed_exit: None,
};
let answer = crate::SessionDescription::try_from("synthetic-answer".to_owned()).unwrap();
let mut connection = Box::pin(host.apply_answer(answer));
@@ -110,3 +112,32 @@ async fn negotiation_timeout_waits_for_helper_exit_before_allowing_retry() -> an
);
Ok(())
}
+
+#[cfg(unix)]
+#[tokio::test]
+async fn close_after_output_eof_reuses_observed_exit() -> anyhow::Result<()> {
+ let spawned = codex_utils_pty::spawn_pipe_process(
+ std::path::Path::new("/bin/sleep"),
+ &["30".to_owned()],
+ std::path::Path::new("/"),
+ &child_environment(std::iter::empty()),
+ /*arg0*/ &None,
+ &[],
+ )
+ .await?;
+ let (output, receiver) = tokio::sync::mpsc::channel(/*buffer*/ 1);
+ drop(output);
+ let (reaped, exit) = tokio::sync::oneshot::channel();
+ let exit_code = crate::HelperExitStage::AudioService.code();
+ reaped.send(exit_code).unwrap();
+ let mut host = super::VoiceHost {
+ process: spawned.session,
+ output: crate::message_reader::MessageReader::new(receiver),
+ exit,
+ observed_exit: None,
+ };
+ assert!(host.next_response().await.is_err());
+ assert_eq!(host.observed_exit, Some(Some(exit_code)));
+ assert!(host.close().await.is_err());
+ Ok(())
+}
diff --git a/codex-rs/realtime-webrtc/src/helper_exit.rs b/codex-rs/realtime-webrtc/src/helper_exit.rs
new file mode 100644
index 0000000000..36c23f79e5
--- /dev/null
+++ b/codex-rs/realtime-webrtc/src/helper_exit.rs
@@ -0,0 +1,55 @@
+//! Fixed helper exit phases for safe diagnostics when its framed output closes.
+//! The same-build handshake keeps these codes aligned between parent and helper.
+
+#[derive(Clone, Copy, Debug)]
+#[repr(i32)]
+pub enum HelperExitStage {
+ ControlRead = 20,
+ ControlQueue = 21,
+ ParentGone = 22,
+ Runtime = 23,
+ Transport = 24,
+ OpenDevices = 25,
+ AudioIngress = 26,
+ AudioService = 27,
+ InspectAudio = 28,
+ AudioControls = 29,
+ Reply = 30,
+ Shutdown = 31,
+ ControlSequence = 32,
+ Playout = 33,
+ Render = 35,
+ Capture = 36,
+ Device = 37,
+ Send = 39,
+}
+
+impl HelperExitStage {
+ pub fn code(self) -> i32 {
+ self as i32
+ }
+
+ pub fn from_code(code: i32) -> Option<Self> {
+ Some(match code {
+ 20 => Self::ControlRead,
+ 21 => Self::ControlQueue,
+ 22 => Self::ParentGone,
+ 23 => Self::Runtime,
+ 24 => Self::Transport,
+ 25 => Self::OpenDevices,
+ 26 => Self::AudioIngress,
+ 27 => Self::AudioService,
+ 28 => Self::InspectAudio,
+ 29 => Self::AudioControls,
+ 30 => Self::Reply,
+ 31 => Self::Shutdown,
+ 32 => Self::ControlSequence,
+ 33 => Self::Playout,
+ 35 => Self::Render,
+ 36 => Self::Capture,
+ 37 => Self::Device,
+ 39 => Self::Send,
+ _ => return None,
+ })
+ }
+}
diff --git a/codex-rs/realtime-webrtc/src/lib.rs b/codex-rs/realtime-webrtc/src/lib.rs
index 8abfa53eb3..6ce6b1d732 100644
--- a/codex-rs/realtime-webrtc/src/lib.rs
+++ b/codex-rs/realtime-webrtc/src/lib.rs
@@ -1,10 +1,12 @@
mod client;
+mod helper_exit;
mod message_reader;
mod protocol;
mod session;
pub use client::ConnectionError;
pub use client::VoiceHost;
+pub use helper_exit::HelperExitStage;
pub use protocol::AudioControls;
pub use protocol::AudioState;
pub use protocol::MAX_FRAME_BYTES;
diff --git a/codex-rs/realtime-webrtc/src/session.rs b/codex-rs/realtime-webrtc/src/session.rs
index b699c03e4c..a0f95b4bfe 100644
--- a/codex-rs/realtime-webrtc/src/session.rs
+++ b/codex-rs/realtime-webrtc/src/session.rs
@@ -133,11 +133,14 @@ impl RealtimeWebrtcSession {
.name("voice-session".into())
.spawn(move || {
let task = async {
- let host = VoiceHost::connect(&package, &build_commit)
- .await?
- .initialize_runtime()
- .await?;
- let (host, sdp) = host.start_transport().await?;
+ let host = report_failure(
+ "connect",
+ VoiceHost::connect(&package, &build_commit).await,
+ )?;
+ let host =
+ report_failure("initialize_runtime", host.initialize_runtime().await)?;
+ let (host, sdp) =
+ report_failure("start_transport", host.start_transport().await)?;
offer
.send(sdp.into_sdp())
.map_err(|_| anyhow::anyhow!("voice startup cancelled"))?;
@@ -274,24 +277,25 @@ async fn run(
return Ok(());
}
};
- host = host.open_devices().await?;
+ host = report_failure("open_devices", host.open_devices().await)?;
let applied = startup_controls(&mut commands, controls, |initial| {
host.begin_audio_controls(initial)
- })?;
- applied.await?;
+ });
+ let applied = report_failure("queue_startup_controls", applied)?;
+ report_failure("apply_startup_controls", applied.await)?;
connected = true;
let _ = complete.send(Ok(()));
}
Some(Command::Controls(next)) => {
if connected {
- host.set_audio_controls(next).await?;
+ report_failure("set_audio_controls", host.set_audio_controls(next).await)?;
}
}
Some(Command::Answer(..)) => anyhow::bail!("voice answer already applied"),
- None => return host.close().await,
+ None => return report_failure("close", host.close().await),
},
_ = poll.tick() => {
- let audio = host.inspect_audio().await?;
+ let audio = report_failure("inspect_audio", host.inspect_audio().await)?;
state.microphone.fetch_max(audio.microphone_peak, Ordering::Release);
state.speaker.fetch_max(audio.speaker_peak, Ordering::Release);
}
@@ -299,6 +303,26 @@ async fn run(
}
}
+// Keep diagnostics bounded and independent of untyped native, SDP, or device error text.
+fn report_failure<T>(stage: &'static str, result: Result<T>) -> Result<T> {
+ result.inspect_err(|error| {
+ let kind = if error.is::<tokio::time::error::Elapsed>() {
+ "timeout"
+ } else if let Some(error) = error.downcast_ref::<std::io::Error>() {
+ match error.kind() {
+ std::io::ErrorKind::UnexpectedEof
+ | std::io::ErrorKind::BrokenPipe
+ | std::io::ErrorKind::ConnectionReset => "closed",
+ std::io::ErrorKind::InvalidData => "protocol",
+ _ => "io",
+ }
+ } else {
+ "other"
+ };
+ tracing::warn!(stage, kind, "voice session operation failed");
+ })
+}
+
// Devices are still disabled. The snapshot and request enqueue share the setters' lock;
// later setters remain ordered after this request without waiting for its acknowledgement.
fn startup_controls<T>(
diff --git a/codex-rs/tui/src/startup_orchestration.rs b/codex-rs/tui/src/startup_orchestration.rs
index 3ccc92c4f9..db0e05f63b 100644
--- a/codex-rs/tui/src/startup_orchestration.rs
+++ b/codex-rs/tui/src/startup_orchestration.rs
@@ -562,7 +562,9 @@ pub(super) async fn run_main_inner(
let log_file = log_file_opts.open(log_dir.join(TUI_LOG_FILE_NAME))?;
let (non_blocking, guard) = non_blocking(log_file);
let env_filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| {
- EnvFilter::new("codex_core=info,codex_tui=info,codex_rmcp_client=info")
+ EnvFilter::new(
+ "codex_core=info,codex_tui=info,codex_rmcp_client=info,codex_realtime_webrtc=warn",
+ )
});
let file_layer = tracing_subscriber::fmt::layer()
.with_writer(non_blocking)
diff --git a/codex-rs/voice-host/src/audio_track.rs b/codex-rs/voice-host/src/audio_track.rs
index 1d3de94890..1525465e7d 100644
--- a/codex-rs/voice-host/src/audio_track.rs
+++ b/codex-rs/voice-host/src/audio_track.rs
@@ -1,4 +1,5 @@
-//! A single Opus RTP track. Elapsed mute time advances its clock without inventing packet loss.
+//! A single Opus RTP track. Muted capture sends generated silence to keep the peer alive;
+//! elapsed time still advances its clock without inventing packet loss.
use std::sync::Arc;
use std::time::Duration;
@@ -26,6 +27,7 @@ pub(crate) struct AudioTrack {
pub(crate) track: Arc<TrackLocalStaticSample>,
ssrc: u32,
end: Option<Instant>,
+ silence: Option<Vec<u8>>,
}
impl AudioTrack {
@@ -70,10 +72,47 @@ impl AudioTrack {
track: Arc::new(track),
ssrc,
end: None,
+ silence: None,
},
))
}
+ /// Send only synthetic silence while muted, never a device or processing buffer.
+ /// Reuse the encoded packet and pace it by the same RTP clock as live capture.
+ #[cfg(any(
+ target_os = "macos",
+ all(target_os = "linux", target_env = "gnu"),
+ all(windows, target_env = "msvc")
+ ))]
+ pub(crate) async fn send_muted_silence(&mut self, at: Instant) -> Result<(), &'static str> {
+ if self.end.is_some_and(|end| at < end) {
+ return Ok(());
+ }
+ if self.silence.is_none() {
+ let mut encoder = opus::Encoder::new(
+ /*sample_rate*/ 48_000,
+ opus::Channels::Mono,
+ opus::Application::Voip,
+ )
+ .map_err(|_| "failed to create voice silence encoder")?;
+ let mut data = vec![0; 1275];
+ let len = encoder
+ .encode_float(&[0.0; 960], &mut data)
+ .map_err(|_| "failed to encode voice silence")?;
+ data.truncate(len);
+ self.silence = Some(data);
+ }
+ self.send(EncodedAudio {
+ data: self
+ .silence
+ .as_ref()
+ .ok_or("voice silence unavailable")?
+ .clone(),
+ at,
+ })
+ .await
+ }
+
pub(crate) async fn send(&mut self, frame: EncodedAudio) -> Result<(), &'static str> {
tokio::time::timeout(SEND_TIMEOUT, async {
let mut gap = self
diff --git a/codex-rs/voice-host/src/capture_worker.rs b/codex-rs/voice-host/src/capture_worker.rs
index c97ab53f33..dc6f8ef47d 100644
--- a/codex-rs/voice-host/src/capture_worker.rs
+++ b/codex-rs/voice-host/src/capture_worker.rs
@@ -1,5 +1,6 @@
//! Pumps bounded device queues through capture processing and the outgoing audio track.
//! Devices retain real stream ownership; all queued and partial capture obey mute generations.
+//! Muted sessions send generated Opus silence independently of device capture.
//! Retain at most one encoded batch and return between packets so controls can take priority.
use std::collections::VecDeque;
@@ -11,6 +12,7 @@ use std::time::Instant;
use super::MAX_CAPTURE_AGE;
use super::buffers::Buffers;
use super::processing;
+use crate::service_failure::ServiceFailure;
pub(super) struct CaptureWorker {
pub(super) buffers: Arc<Buffers>,
@@ -43,36 +45,90 @@ impl CaptureWorker {
// Some backends start callbacks during stream construction, before open returns.
self.buffers.serviced.store(true, Ordering::Release);
let generation = self.buffers.microphone.load(Ordering::Acquire);
+ if self.buffers.render_dropped.swap(false, Ordering::AcqRel) {
+ self.processor.reset_render();
+ for _ in 0..self.buffers.rendered.capacity() {
+ if self.buffers.rendered.pop().is_none() {
+ break;
+ }
+ }
+ }
+ if self.buffers.capture_dropped.swap(false, Ordering::AcqRel) {
+ self.pending.clear();
+ self.processor.reset().map_err(io::Error::other)?;
+ for _ in 0..self.buffers.capture.capacity() {
+ if self.buffers.capture.pop().is_none() {
+ break;
+ }
+ }
+ }
+ let mut render_stale = false;
for _ in 0..self.buffers.rendered.capacity() {
let Some(frame) = self.buffers.rendered.pop() else {
break;
};
if frame.at.elapsed() > MAX_CAPTURE_AGE {
- return Err(io::Error::other("audio rendering fell behind"));
+ if !render_stale {
+ self.processor.reset_render();
+ render_stale = true;
+ }
+ continue;
}
- self.processor.render(&frame).map_err(io::Error::other)?;
+ render_stale = false;
+ self.processor
+ .render(&frame)
+ .map_err(|_| io::Error::other(ServiceFailure::Render))?;
}
if self.pending.is_empty() {
+ let mut capture_stale = false;
for _ in 0..self.buffers.capture.capacity() {
let Some(frame) = self.buffers.capture.pop() else {
break;
};
if generation.is_multiple_of(2) && frame.generation == generation {
+ if now().saturating_duration_since(frame.at) > processing::MAX_PROCESSING_DELAY
+ {
+ self.pending.clear();
+ if !capture_stale {
+ self.processor.reset().map_err(io::Error::other)?;
+ capture_stale = true;
+ }
+ continue;
+ }
+ capture_stale = false;
let frames = self.processor.capture(&frame, &now);
- self.pending.extend(frames.map_err(io::Error::other)?);
+ match frames {
+ Ok(frames) => self.pending.extend(frames),
+ Err(processing::PROCESSING_LATE) => {
+ self.pending.clear();
+ self.processor.reset().map_err(io::Error::other)?;
+ }
+ Err(_) => return Err(io::Error::other(ServiceFailure::Capture)),
+ }
}
}
}
if self.buffers.failed.load(Ordering::Acquire) {
- return Err(io::Error::other("audio device failed"));
+ return Err(io::Error::other(ServiceFailure::Device));
}
let Some(frame) = self.pending.pop_front() else {
+ if !generation.is_multiple_of(2) {
+ audio
+ .send_muted_silence(now())
+ .await
+ .map_err(|_| io::Error::other(ServiceFailure::Send))?;
+ }
return Ok(0);
};
if now().saturating_duration_since(frame.at) > processing::MAX_PROCESSING_DELAY {
- return Err(io::Error::other("voice processing fell behind"));
+ self.pending.clear();
+ self.processor.reset().map_err(io::Error::other)?;
+ return Ok(0);
}
- audio.send(frame).await.map_err(io::Error::other)?;
+ audio
+ .send(frame)
+ .await
+ .map_err(|_| io::Error::other(ServiceFailure::Send))?;
Ok(1)
}
}
diff --git a/codex-rs/voice-host/src/capture_worker_tests.rs b/codex-rs/voice-host/src/capture_worker_tests.rs
index ef18870307..17e739b0bf 100644
--- a/codex-rs/voice-host/src/capture_worker_tests.rs
+++ b/codex-rs/voice-host/src/capture_worker_tests.rs
@@ -267,12 +267,32 @@ async fn capture_reaches_remote_rtp_and_mute_discards_queued_and_partial_audio()
worker.set_controls(muted()).unwrap();
worker.set_controls(unmuted()).unwrap();
let old_generation = worker.buffers.microphone.load(Ordering::Acquire);
- now = Instant::now();
+ now = Instant::now().max(start + Duration::from_millis(/*millis*/ 100));
enqueue(&worker, /*samples*/ 720, old_generation, now);
assert_eq!(worker.service(&mut local.audio, || now).await.unwrap(), 0);
enqueue(&worker, /*samples*/ 2_880, old_generation, now);
worker.set_controls(muted()).unwrap();
assert_eq!(worker.service(&mut local.audio, || now).await.unwrap(), 0);
+ let silence = received.recv().await.unwrap();
+ let mut decoder = opus::Decoder::new(/*sample_rate*/ 48_000, opus::Channels::Mono).unwrap();
+ let mut decoded = [1.0; 960];
+ assert_eq!(
+ decoder
+ .decode_float(&silence.payload, &mut decoded, /*fec*/ false)
+ .unwrap(),
+ 960
+ );
+ assert!(decoded.iter().all(|sample| sample.abs() < 0.0001));
+ assert_eq!(
+ silence.header.sequence_number,
+ before
+ .last()
+ .unwrap()
+ .header
+ .sequence_number
+ .wrapping_add(1)
+ );
+ assert!(received.try_recv().is_err());
// A callback that began before mute may enqueue its old generation only
// after unmute; a newly started callback may still carry an old device
@@ -304,12 +324,7 @@ async fn capture_reaches_remote_rtp_and_mute_discards_queued_and_partial_audio()
);
assert_eq!(
after[0].header.sequence_number,
- before
- .last()
- .unwrap()
- .header
- .sequence_number
- .wrapping_add(1)
+ silence.header.sequence_number.wrapping_add(1)
);
assert!(received.try_recv().is_err());
@@ -337,22 +352,77 @@ async fn capture_reaches_remote_rtp_and_mute_discards_queued_and_partial_audio()
assert_eq!(worker.service(&mut local.audio, || now).await.unwrap(), 0);
assert!(received.try_recv().is_err());
- // Waiting behind a slow send cannot extend the capture freshness limit.
+ // Waiting behind a slow send discards stale audio without ending the session.
let generation = worker.buffers.microphone.load(Ordering::Acquire);
now = Instant::now() + Duration::from_secs(/*secs*/ 1);
enqueue(&worker, /*samples*/ 2_880, generation, now);
assert_eq!(worker.service(&mut local.audio, || now).await.unwrap(), 1);
received.recv().await.unwrap();
let stale = now + Duration::from_secs(/*secs*/ 1);
+ assert_eq!(worker.service(&mut local.audio, || stale).await.unwrap(), 0);
+ assert!(received.try_recv().is_err());
+ now = stale + Duration::from_millis(/*millis*/ 10);
+ enqueue(&worker, /*samples*/ 2_880, generation, now);
+ let resumed = receive_capture(&mut worker, &mut local, &mut received, now).await;
+ assert!(!resumed.is_empty());
+
+ // A saturated callback queue also retires partial processing state, then
+ // admits new capture once the worker catches up.
+ enqueue(&worker, /*samples*/ 2_880, generation, now);
+ worker
+ .buffers
+ .capture_dropped
+ .store(true, Ordering::Release);
+ assert_eq!(worker.service(&mut local.audio, || now).await.unwrap(), 0);
+ assert!(received.try_recv().is_err());
+ now += Duration::from_millis(/*millis*/ 70);
+ enqueue(&worker, /*samples*/ 2_880, generation, now);
+ assert!(
+ !receive_capture(&mut worker, &mut local, &mut received, now)
+ .await
+ .is_empty()
+ );
+
+ // A delayed worker must discard a full render-reference queue before
+ // it feeds stale speaker audio back into echo cancellation.
+ let mut output = [0.0_f32; BLOCK];
+ let mut output_state = super::super::OutputState::default();
+ for _ in 0..=worker.buffers.rendered.capacity() {
+ super::super::render_output(
+ &mut output,
+ /*channels*/ 1,
+ /*rate*/ 44_100.0,
+ now,
+ &worker.buffers,
+ &mut output_state,
+ );
+ }
assert_eq!(
- worker
- .service(&mut local.audio, || stale)
+ worker.buffers.rendered.len(),
+ worker.buffers.rendered.capacity()
+ );
+ assert!(worker.buffers.render_dropped.load(Ordering::Acquire));
+ assert_eq!(worker.service(&mut local.audio, || now).await.unwrap(), 0);
+ assert!(worker.buffers.rendered.is_empty());
+ assert!(!worker.buffers.render_dropped.load(Ordering::Acquire));
+ assert!(!worker.buffers.failed.load(Ordering::Acquire));
+
+ now += Duration::from_millis(/*millis*/ 70);
+ super::super::render_output(
+ &mut output,
+ /*channels*/ 1,
+ /*rate*/ 44_100.0,
+ now,
+ &worker.buffers,
+ &mut output_state,
+ );
+ enqueue(&worker, /*samples*/ 2_880, generation, now);
+ assert!(
+ !receive_capture(&mut worker, &mut local, &mut received, now)
.await
- .unwrap_err()
- .to_string(),
- "voice processing fell behind"
+ .is_empty()
);
- assert!(received.try_recv().is_err());
+ assert!(worker.buffers.rendered.is_empty());
local.close().await.unwrap();
remote.close().await.unwrap();
})
diff --git a/codex-rs/voice-host/src/device_buffers.rs b/codex-rs/voice-host/src/device_buffers.rs
index 0609d9dbae..b38f13fe2e 100644
--- a/codex-rs/voice-host/src/device_buffers.rs
+++ b/codex-rs/voice-host/src/device_buffers.rs
@@ -98,6 +98,8 @@ pub(super) struct Buffers {
pub(super) speaker_failure_gate: Mutex<()>,
pub(super) serviced: AtomicBool,
pub(super) failed: AtomicBool,
+ pub(super) capture_dropped: AtomicBool,
+ pub(super) render_dropped: AtomicBool,
pub(super) microphone_peak: AtomicU16,
pub(super) speaker_peak: AtomicU16,
pub(super) queued: AtomicU32,
@@ -136,6 +138,8 @@ impl Buffers {
speaker_failure_gate: Mutex::new(()),
serviced: AtomicBool::new(false),
failed: AtomicBool::new(false),
+ capture_dropped: AtomicBool::new(false),
+ render_dropped: AtomicBool::new(false),
microphone_peak: AtomicU16::new(/*v*/ 0),
speaker_peak: AtomicU16::new(/*v*/ 0),
queued: AtomicU32::new(/*v*/ 0),
diff --git a/codex-rs/voice-host/src/devices.rs b/codex-rs/voice-host/src/devices.rs
index 5fcaab5232..1f1d58f1f1 100644
--- a/codex-rs/voice-host/src/devices.rs
+++ b/codex-rs/voice-host/src/devices.rs
@@ -1,5 +1,5 @@
//! Owns local device streams on the helper worker. Callbacks allocate no buffers and take no locks.
-//! Small callbacks share full queue slots; processing lag still fails the session closed.
+//! Small callbacks share full queue slots; overload discards stale media and resumes fresh audio.
//! Capture and actual rendered output carry device timing.
//! References start with worker service; unmute rejects earlier device capture buffers.
@@ -207,7 +207,9 @@ impl Devices {
audio: &mut crate::audio_track::AudioTrack,
) -> io::Result<usize> {
if let Some(playout) = &self.playout {
- playout.check().map_err(io::Error::other)?;
+ playout
+ .check()
+ .map_err(|_| io::Error::other(crate::service_failure::ServiceFailure::Playout))?;
}
self.worker.service(audio, Instant::now).await
}
@@ -338,7 +340,8 @@ where
record_peak(&buffers.microphone_peak, *output);
}
if !capture.push(frame, rate, &buffers.capture) {
- buffers.failed.store(true, Ordering::Release);
+ capture.reset();
+ buffers.capture_dropped.store(true, Ordering::Release);
return;
}
}
@@ -429,7 +432,8 @@ fn render_output<T>(
record_peak(&buffers.speaker_peak, *sample);
}
if !output.reference.push(reference, rate, &buffers.rendered) {
- buffers.failed.store(true, Ordering::Release);
+ output.reference.reset();
+ buffers.render_dropped.store(true, Ordering::Release);
}
}
if let Some(end) = delivered_until {
diff --git a/codex-rs/voice-host/src/devices_tests.rs b/codex-rs/voice-host/src/devices_tests.rs
index f313fe61f1..2bfcffee77 100644
--- a/codex-rs/voice-host/src/devices_tests.rs
+++ b/codex-rs/voice-host/src/devices_tests.rs
@@ -2,7 +2,7 @@ use super::*;
use pretty_assertions::assert_eq;
#[test]
-fn startup_output_waits_for_service_but_later_overflow_still_fails() {
+fn startup_output_waits_for_service_and_later_overflow_discards_reference() {
let buffers = Buffers::new(/*input_rate*/ 48_000, /*output_rate*/ 384_000);
let mut state = OutputState::default();
let mut output = [1.0_f32; BLOCK * 2];
@@ -65,8 +65,9 @@ fn startup_output_waits_for_service_but_later_overflow_still_fails() {
&buffers,
&mut state,
);
- assert!(buffers.failed.load(Ordering::Acquire));
- assert!(buffers.take_state().is_err());
+ assert!(!buffers.failed.load(Ordering::Acquire));
+ assert!(buffers.render_dropped.load(Ordering::Acquire));
+ assert!(buffers.take_state().is_ok());
}
#[test]
@@ -332,7 +333,8 @@ fn suppressed_output_does_not_fail_when_reference_queue_is_full() {
&buffers,
&mut state,
);
- assert!(buffers.failed.load(Ordering::Acquire));
+ assert!(!buffers.failed.load(Ordering::Acquire));
+ assert!(buffers.render_dropped.load(Ordering::Acquire));
}
#[test]
diff --git a/codex-rs/voice-host/src/incoming.rs b/codex-rs/voice-host/src/incoming.rs
index b230d0eda5..89c46d1fff 100644
--- a/codex-rs/voice-host/src/incoming.rs
+++ b/codex-rs/voice-host/src/incoming.rs
@@ -1,5 +1,6 @@
//! Takes incoming Opus RTP before the upstream track queue, preserving network arrival time.
//! Packet and byte permits follow owned data through the consumer, not merely through this queue.
+//! Stale or saturated media is discarded so a slow worker can resume with fresh speaker audio.
use std::sync::Arc;
use std::sync::atomic::AtomicBool;
@@ -106,7 +107,7 @@ impl Incoming {
};
if epoch.is_multiple_of(2) && packet.epoch == epoch {
if packet.at.elapsed() > Duration::from_secs(/*secs*/ 1) {
- return Err("incoming audio fell behind");
+ continue;
}
return Ok(Some(packet));
}
@@ -127,38 +128,35 @@ impl Ingress {
return Ok(());
}
let size = packet.marshal_size();
- let accepted = (|| {
- if size > PACKET_BYTES
- || packet.header.payload_type != crate::audio_track::OPUS_PAYLOAD_TYPE
- || self.stream.is_some_and(|ssrc| ssrc != packet.header.ssrc)
- {
- return None;
- }
- self.stream = Some(packet.header.ssrc);
- let packet_permit = self.state.packets.clone().try_acquire_owned().ok()?;
- let bytes_permit = self
- .state
- .bytes
- .clone()
- .try_acquire_many_owned(size as u32)
- .ok()?;
- let mut data = vec![0; size];
- if packet.marshal_to(&mut data).ok()? != size {
- return None;
- }
- self.sender
- .try_send(ReceivedRtp {
- data,
- at: message.now,
- epoch,
- _packet: packet_permit,
- _bytes: bytes_permit,
- })
- .ok()
- })();
- if accepted.is_none() {
+ if size > PACKET_BYTES
+ || packet.header.payload_type != crate::audio_track::OPUS_PAYLOAD_TYPE
+ || self.stream.is_some_and(|ssrc| ssrc != packet.header.ssrc)
+ {
self.state.failed.store(true, Ordering::Release);
+ return Ok(());
}
+ self.stream = Some(packet.header.ssrc);
+ if message.now.elapsed() > Duration::from_secs(/*secs*/ 1) {
+ return Ok(());
+ }
+ let Ok(packet_permit) = self.state.packets.clone().try_acquire_owned() else {
+ return Ok(());
+ };
+ let Ok(bytes_permit) = self.state.bytes.clone().try_acquire_many_owned(size as u32) else {
+ return Ok(());
+ };
+ let mut data = vec![0; size];
+ if packet.marshal_to(&mut data).ok() != Some(size) {
+ self.state.failed.store(true, Ordering::Release);
+ return Ok(());
+ }
+ let _ = self.sender.try_send(ReceivedRtp {
+ data,
+ at: message.now,
+ epoch,
+ _packet: packet_permit,
+ _bytes: bytes_permit,
+ });
// Consume here: no second copy is retained by the upstream track-event queue.
Ok(())
}
diff --git a/codex-rs/voice-host/src/incoming_tests.rs b/codex-rs/voice-host/src/incoming_tests.rs
index bc32d83845..70d6048e21 100644
--- a/codex-rs/voice-host/src/incoming_tests.rs
+++ b/codex-rs/voice-host/src/incoming_tests.rs
@@ -41,7 +41,7 @@ fn arrival_and_rtp_bytes_survive_the_adapter_without_an_upstream_copy() {
}
#[test]
-fn consumed_packets_keep_their_budget_until_the_last_owner_drops_them() {
+fn exhausted_packet_or_byte_budget_drops_audio_then_accepts_fresh_packets() {
for size in [32, PACKET_BYTES] {
let (mut incoming, mut ingress) = pair();
let count = PACKETS.min(BYTES / size);
@@ -54,8 +54,11 @@ fn consumed_packets_keep_their_budget_until_the_last_owner_drops_them() {
ingress.handle_read(packet(size, Instant::now())).unwrap();
held.push(incoming.take().unwrap().unwrap());
ingress.handle_read(packet(size, Instant::now())).unwrap();
- assert_eq!(incoming.take().err(), Some("incoming audio failed"));
+ assert!(incoming.take().unwrap().is_none());
+ assert!(!incoming.state.failed.load(Ordering::Acquire));
drop(held);
+ ingress.handle_read(packet(size, Instant::now())).unwrap();
+ assert!(incoming.take().unwrap().is_some());
assert_eq!(
(
incoming.state.packets.available_permits(),
@@ -66,6 +69,22 @@ fn consumed_packets_keep_their_budget_until_the_last_owner_drops_them() {
}
}
+#[test]
+fn old_speaker_packets_are_discarded_without_poisoning_fresh_audio() {
+ let (mut incoming, mut ingress) = pair();
+ ingress
+ .handle_read(packet(/*size*/ 32, Instant::now()))
+ .unwrap();
+ let mut queued = incoming.receiver.try_recv().unwrap();
+ queued.at -= Duration::from_secs(/*secs*/ 2);
+ ingress.sender.try_send(queued).unwrap();
+ assert!(incoming.take().unwrap().is_none());
+ ingress
+ .handle_read(packet(/*size*/ 32, Instant::now()))
+ .unwrap();
+ assert!(incoming.take().unwrap().is_some());
+}
+
#[test]
fn suppression_discards_queued_and_in_flight_old_epochs() {
let (mut incoming, mut ingress) = pair();
diff --git a/codex-rs/voice-host/src/main.rs b/codex-rs/voice-host/src/main.rs
index 20c252eb21..709521aa0e 100644
--- a/codex-rs/voice-host/src/main.rs
+++ b/codex-rs/voice-host/src/main.rs
@@ -22,17 +22,21 @@ mod audio_track;
mod devices;
mod incoming;
mod runtime;
+mod service_failure;
mod transport;
mod transport_runtime;
+use std::cell::Cell;
use std::io;
use std::io::Write;
use std::sync::mpsc;
use std::time::Duration;
+use codex_realtime_webrtc::HelperExitStage;
use codex_realtime_webrtc::Message;
use codex_realtime_webrtc::encode_frame;
use codex_realtime_webrtc::read_message;
+use service_failure::ServiceFailure;
const DEVICE_SERVICE_INTERVAL: Duration = Duration::from_millis(/*millis*/ 5);
@@ -47,14 +51,16 @@ fn main() {
match (args.next(), args.next()) {
(Some(arg), None) if arg == "--build-commit" => println!("{BUILD_COMMIT}"),
(None, None) => {
- if run(|executor| {
+ let phase = Cell::new(HelperExitStage::ControlSequence);
+ if run(&phase, |executor| {
+ phase.set(HelperExitStage::Transport);
executor
.block_on(transport::Transport::new())
.map_err(io::Error::other)
})
.is_err()
{
- std::process::exit(/*code*/ 1);
+ std::process::exit(phase.get().code());
}
}
_ => std::process::exit(/*code*/ 2),
@@ -62,6 +68,7 @@ fn main() {
}
fn run(
+ phase: &Cell<HelperExitStage>,
start_transport: impl Fn(&tokio::runtime::Runtime) -> io::Result<transport::Transport>,
) -> io::Result<()> {
let (sender, receiver) = mpsc::sync_channel(/*bound*/ 1);
@@ -73,17 +80,17 @@ fn run(
match read_message(&mut input) {
Ok(Some(message)) => {
if sender.try_send(message).is_err() {
- std::process::exit(/*code*/ 1);
+ std::process::exit(HelperExitStage::ControlQueue.code());
}
}
Ok(None) => break,
- Err(_) => std::process::exit(/*code*/ 1),
+ Err(_) => std::process::exit(HelperExitStage::ControlRead.code()),
}
}
drop(sender);
// Independent of the main worker or a blocked stdout write, including after parent death.
std::thread::sleep(Duration::from_secs(/*secs*/ 2));
- std::process::exit(/*code*/ 1);
+ std::process::exit(HelperExitStage::ParentGone.code());
})?;
let Ok(hello) = receiver.recv() else {
return Ok(());
@@ -96,6 +103,7 @@ fn run(
{
return Err(io::Error::other("incompatible voice helper"));
}
+ phase.set(HelperExitStage::Reply);
let mut output = io::stdout().lock();
output.write_all(&encode_frame(&Message::Ready {})?)?;
output.flush()?;
@@ -105,6 +113,7 @@ fn run(
let mut answered = false;
let mut devices: Option<devices::Devices> = None;
loop {
+ phase.set(HelperExitStage::ControlSequence);
let message = if let Some(devices) = &mut devices {
let peer = transport
.as_mut()
@@ -113,12 +122,29 @@ fn run(
// Bound one pass to the ingress queue capacity so new media
// cannot indefinitely postpone a waiting privacy control.
for _ in 0..64 {
+ phase.set(HelperExitStage::AudioIngress);
let Some(packet) = peer.incoming.take().map_err(io::Error::other)? else {
break;
};
+ phase.set(HelperExitStage::Playout);
devices.receive(packet)?;
}
- executor.block_on(devices.service(&mut peer.audio))?;
+ phase.set(HelperExitStage::AudioService);
+ if let Err(error) = executor.block_on(devices.service(&mut peer.audio)) {
+ if let Some(failure) = error
+ .get_ref()
+ .and_then(|error| error.downcast_ref::<ServiceFailure>())
+ {
+ phase.set(match failure {
+ ServiceFailure::Playout => HelperExitStage::Playout,
+ ServiceFailure::Render => HelperExitStage::Render,
+ ServiceFailure::Capture => HelperExitStage::Capture,
+ ServiceFailure::Device => HelperExitStage::Device,
+ ServiceFailure::Send => HelperExitStage::Send,
+ });
+ }
+ return Err(error);
+ }
Ok(())
})?
.ok_or(mpsc::RecvTimeoutError::Disconnected)
@@ -129,6 +155,7 @@ fn run(
};
let reply = match message {
Ok(Message::InspectAudio {}) => {
+ phase.set(HelperExitStage::InspectAudio);
if answered && transport.as_ref().is_none_or(|peer| !*peer.ready.borrow()) {
return Err(io::Error::other("voice connection closed"));
}
@@ -136,15 +163,18 @@ fn run(
state: devices
.as_ref()
.map(devices::Devices::take_state)
- .transpose()?
+ .transpose()
+ .inspect_err(|_| phase.set(HelperExitStage::Device))?
.unwrap_or_default(),
}
}
Ok(Message::OpenDevices {}) if devices.is_none() && runtime.is_some() && answered => {
+ phase.set(HelperExitStage::OpenDevices);
devices = Some(devices::Devices::open()?);
Message::DevicesOpened {}
}
Ok(Message::SetAudioControls { controls }) => {
+ phase.set(HelperExitStage::AudioControls);
let peer = transport
.as_ref()
.ok_or_else(|| io::Error::other("voice peer not started"))?;
@@ -165,6 +195,7 @@ fn run(
Message::AudioControlsApplied {}
}
Ok(Message::StartTransport {}) if transport.is_none() => {
+ phase.set(HelperExitStage::Transport);
let peer = start_transport(&executor)?;
let sdp = executor.block_on(peer.offer()).map_err(io::Error::other)?;
transport = Some(peer);
@@ -173,6 +204,7 @@ fn run(
}
}
Ok(Message::ApplyAnswer { sdp }) if !answered => {
+ phase.set(HelperExitStage::Transport);
let Some(peer) = transport.as_ref() else {
return Err(io::Error::other("voice transport not started"));
};
@@ -189,6 +221,7 @@ fn run(
Message::TransportReady {}
}
Ok(Message::InitializeRuntime {}) => {
+ phase.set(HelperExitStage::Runtime);
if runtime.is_some() {
return Err(io::Error::other("runtime already initialized"));
}
@@ -196,6 +229,7 @@ fn run(
Message::RuntimeReady {}
}
Ok(Message::Close {}) => {
+ phase.set(HelperExitStage::Shutdown);
let _ = devices.take();
if let Some(mut peer) = transport.take() {
executor.block_on(peer.close()).map_err(io::Error::other)?;
@@ -221,6 +255,7 @@ fn run(
| Message::Closed {},
) => return Err(io::Error::other("invalid voice control sequence")),
};
+ phase.set(HelperExitStage::Reply);
output.write_all(&encode_frame(&reply)?)?;
output.flush()?;
}
diff --git a/codex-rs/voice-host/src/main_tests.rs b/codex-rs/voice-host/src/main_tests.rs
index eeeb797b34..8dca96433c 100644
--- a/codex-rs/voice-host/src/main_tests.rs
+++ b/codex-rs/voice-host/src/main_tests.rs
@@ -60,7 +60,7 @@ fn blocked_startup_fixture() {
if std::env::var_os(CHILD_ENV).as_deref() != Some(std::ffi::OsStr::new("1")) {
return;
}
- run(|_| {
+ run(&Cell::new(HelperExitStage::Transport), |_| {
io::stderr().write_all(STARTUP_BLOCKED)?;
io::stderr().flush()?;
loop {
@@ -121,7 +121,10 @@ async fn parent_pipe_loss_terminates_helper_with_blocked_transport_startup() {
assert!(child.try_wait().unwrap().is_none());
drop(input);
let output = child.wait_with_output().await.unwrap();
- assert_eq!(output.status.code(), Some(1));
+ assert_eq!(
+ output.status.code(),
+ Some(HelperExitStage::ParentGone.code())
+ );
assert_eq!((output.stdout, output.stderr), (vec![], vec![]));
})
.await
diff --git a/codex-rs/voice-host/src/playback.rs b/codex-rs/voice-host/src/playback.rs
index 8779c48bff..282f32aa5d 100644
--- a/codex-rs/voice-host/src/playback.rs
+++ b/codex-rs/voice-host/src/playback.rs
@@ -102,7 +102,9 @@ impl PlaybackWriter {
return Ok(bytes);
}
if Instant::now() >= deadline {
- return Err("speaker fell behind");
+ // The sink must consume this chunk to catch up with the device clock.
+ // Dropping it keeps the bounded queue and current epoch intact.
+ return Ok(frame.len * 4);
}
std::thread::park_timeout(Duration::from_millis(/*millis*/ 1));
}
diff --git a/codex-rs/voice-host/src/playback_tests.rs b/codex-rs/voice-host/src/playback_tests.rs
index 18d2796ee5..a3a45d7db8 100644
--- a/codex-rs/voice-host/src/playback_tests.rs
+++ b/codex-rs/voice-host/src/playback_tests.rs
@@ -123,16 +123,19 @@ fn delay_contention_preserves_the_last_snapshot_without_failing_the_device() {
}
#[test]
-fn invalid_samples_and_stalled_consumption_fail_without_growing_the_queue() {
+fn invalid_samples_fail_but_stalled_consumption_drops_and_resumes() {
let (buffers, _, writer) = active(/*rate*/ 8000);
assert_eq!(
writer.write(&f32::NAN.to_le_bytes()),
Err("invalid speaker sample")
);
writer.write(&vec![0; BLOCK * 4]).unwrap();
- assert_eq!(
- writer.write(&vec![0; BLOCK * 4]),
- Err("speaker fell behind")
- );
+ assert_eq!(writer.write(&vec![0; BLOCK * 4]), Ok(BLOCK * 4));
assert_eq!(buffers.queued.load(Ordering::Acquire), BLOCK as u32);
+ let mut playback = Playback::default();
+ for _ in 0..BLOCK {
+ assert!(playback.next(&buffers).is_some());
+ }
+ assert_eq!(writer.write(&vec![0; BLOCK * 4]), Ok(BLOCK * 4));
+ assert!(!buffers.failed.load(Ordering::Acquire));
}
diff --git a/codex-rs/voice-host/src/processing.rs b/codex-rs/voice-host/src/processing.rs
index 4ba26cfb69..b05dd19365 100644
--- a/codex-rs/voice-host/src/processing.rs
+++ b/codex-rs/voice-host/src/processing.rs
@@ -21,6 +21,7 @@ use crate::audio_track::EncodedAudio;
type Result<T> = std::result::Result<T, &'static str>;
const BLOCK: usize = 480;
pub(super) const MAX_PROCESSING_DELAY: Duration = Duration::from_millis(/*millis*/ 500);
+pub(super) const PROCESSING_LATE: &str = "voice processing fell behind";
struct Converter {
resampler: rubato::Async<f32>,
@@ -220,11 +221,11 @@ impl Processor {
while let Some((at, input)) = self.capture.next() {
let capture_age = now().saturating_duration_since(at);
if capture_age > MAX_PROCESSING_DELAY {
- return Err("voice processing fell behind");
+ return Err(PROCESSING_LATE);
}
let delay = (capture_age.as_millis() as i64 + self.render_delay).max(0);
if delay > MAX_PROCESSING_DELAY.as_millis() as i64 {
- return Err("voice processing fell behind");
+ return Err(PROCESSING_LATE);
}
self.apm
.set_stream_delay_ms(delay as i32)
diff --git a/codex-rs/voice-host/src/service_failure.rs b/codex-rs/voice-host/src/service_failure.rs
new file mode 100644
index 0000000000..e42b23360e
--- /dev/null
+++ b/codex-rs/voice-host/src/service_failure.rs
@@ -0,0 +1,32 @@
+//! Fixed local media service failures for diagnostics without native error text.
+
+#[derive(Debug)]
+#[cfg_attr(
+ not(any(
+ target_os = "macos",
+ all(target_os = "linux", target_env = "gnu"),
+ all(windows, target_env = "msvc")
+ )),
+ allow(dead_code)
+)]
+pub(crate) enum ServiceFailure {
+ Playout,
+ Render,
+ Capture,
+ Device,
+ Send,
+}
+
+impl std::fmt::Display for ServiceFailure {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ f.write_str(match self {
+ Self::Playout => "audio playout failed",
+ Self::Render => "audio rendering failed",
+ Self::Capture => "audio capture processing failed",
+ Self::Device => "audio device failed",
+ Self::Send => "voice audio send failed",
+ })
+ }
+}
+
+impl std::error::Error for ServiceFailure {}