Skip to content

armgit / codex

Public Imported from GitHub
0

Lightweight coding agent that runs in your terminal

Rust

Add a helper-backed realtime WebRTC session API (#43097)

Benjamin Carlsson64e9a68987b1
diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock
index 6adf98e9b2..f5db14657d 100644
--- a/codex-rs/Cargo.lock
+++ b/codex-rs/Cargo.lock
@@ -4230,8 +4230,12 @@ name = "codex-realtime-webrtc"
version = "0.0.0"
dependencies = [
"anyhow",
+ "codex-build-info",
"codex-install-context",
"codex-utils-pty",
+ "ctor 0.6.3",
+ "futures",
+ "libc",
"pretty_assertions",
"serde",
"serde_json",
@@ -4633,6 +4637,7 @@ dependencies = [
"codex-app-server-client",
"codex-app-server-protocol",
"codex-arg0",
+ "codex-build-info",
"codex-cloud-config",
"codex-config",
"codex-connectors",
diff --git a/codex-rs/realtime-webrtc/Cargo.toml b/codex-rs/realtime-webrtc/Cargo.toml
index 64e9e81873..9042544395 100644
--- a/codex-rs/realtime-webrtc/Cargo.toml
+++ b/codex-rs/realtime-webrtc/Cargo.toml
@@ -11,12 +11,18 @@ doctest = false
workspace = true
[dependencies]
+codex-build-info = { workspace = true }
anyhow = { workspace = true }
codex-install-context = { workspace = true }
codex-utils-pty = { workspace = true }
+futures = { workspace = true, features = ["std"] }
serde = { workspace = true, features = ["derive"] }
serde_json = { workspace = true }
-tokio = { workspace = true, features = ["sync", "time"] }
+tokio = { workspace = true, features = ["macros", "rt-multi-thread", "sync", "time"] }
[dev-dependencies]
+ctor = { workspace = true }
pretty_assertions = { workspace = true }
+
+[target.'cfg(unix)'.dev-dependencies]
+libc = { workspace = true }
diff --git a/codex-rs/realtime-webrtc/src/client.rs b/codex-rs/realtime-webrtc/src/client.rs
index 2398cdcbbc..dc35f137a5 100644
--- a/codex-rs/realtime-webrtc/src/client.rs
+++ b/codex-rs/realtime-webrtc/src/client.rs
@@ -28,6 +28,53 @@ pub struct VoiceHost {
}
impl VoiceHost {
+ /// Open local devices only after answer negotiation. They initially remain muted/suppressed.
+ pub async fn open_devices(mut self) -> Result<Self> {
+ self.exchange(Message::OpenDevices {}, Message::DevicesOpened {}, DEADLINE)
+ .await?;
+ Ok(self)
+ }
+
+ /// Acknowledgement follows invalidation of the helper's previous capture/render generations.
+ pub async fn set_audio_controls(&mut self, controls: crate::AudioControls) -> Result<()> {
+ self.exchange(
+ Message::SetAudioControls { controls },
+ Message::AudioControlsApplied {},
+ DEADLINE,
+ )
+ .await
+ }
+
+ /// Enqueue startup controls synchronously so the facade can order them with setters.
+ /// The returned future owns only the acknowledgement wait, never the facade control lock.
+ pub(crate) fn begin_audio_controls(
+ &mut self,
+ controls: crate::AudioControls,
+ ) -> Result<impl std::future::Future<Output = Result<()>> + '_> {
+ self.process
+ .writer_sender()
+ .try_send(encode_frame(&Message::SetAudioControls { controls })?)
+ .map_err(|_| anyhow::anyhow!("voice helper input unavailable"))?;
+ let deadline = tokio::time::Instant::now() + DEADLINE;
+ Ok(async move {
+ let response = tokio::time::timeout_at(deadline, self.output.next()).await??;
+ ensure!(
+ response == Message::AudioControlsApplied {},
+ "unexpected voice helper response"
+ );
+ Ok(())
+ })
+ }
+
+ /// Consume peaks and detect helper loss even when neither device is producing audio.
+ pub async fn inspect_audio(&mut self) -> Result<crate::AudioState> {
+ let response = self.request(Message::InspectAudio {}, DEADLINE).await?;
+ let Message::AudioState { state } = response else {
+ anyhow::bail!("unexpected voice helper response");
+ };
+ Ok(state)
+ }
+
/// Gather an offer in the helper. This establishes neither connectivity nor audio readiness.
pub async fn start_transport(mut self) -> Result<(Self, crate::SessionDescription)> {
let response = self
diff --git a/codex-rs/realtime-webrtc/src/lib.rs b/codex-rs/realtime-webrtc/src/lib.rs
index 388235be1f..c65d724755 100644
--- a/codex-rs/realtime-webrtc/src/lib.rs
+++ b/codex-rs/realtime-webrtc/src/lib.rs
@@ -1,9 +1,11 @@
mod client;
mod message_reader;
mod protocol;
+mod session;
pub use client::VoiceHost;
pub use protocol::AudioControls;
+pub use protocol::AudioState;
pub use protocol::MAX_FRAME_BYTES;
pub use protocol::Message;
pub use protocol::RUNTIME_ENVIRONMENT;
@@ -11,3 +13,6 @@ pub use protocol::SessionDescription;
pub use protocol::decode_frame;
pub use protocol::encode_frame;
pub use protocol::read_message;
+pub use session::RealtimeWebrtcSession;
+pub use session::RealtimeWebrtcSessionHandle;
+pub use session::StartedRealtimeWebrtcSession;
diff --git a/codex-rs/realtime-webrtc/src/protocol.rs b/codex-rs/realtime-webrtc/src/protocol.rs
index 83e3d574a5..f94bf53bae 100644
--- a/codex-rs/realtime-webrtc/src/protocol.rs
+++ b/codex-rs/realtime-webrtc/src/protocol.rs
@@ -8,13 +8,21 @@ use serde::Serialize;
pub const MAX_FRAME_BYTES: usize = 128 * 1024;
-#[derive(Debug, Deserialize, PartialEq, Serialize)]
+#[derive(Clone, Copy, Debug, Deserialize, PartialEq, Serialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct AudioControls {
pub microphone_muted: bool,
pub speaker_suppressed: bool,
}
+/// Peaks contain levels only, never retained audio or backend messages.
+#[derive(Debug, Default, Deserialize, PartialEq, Serialize)]
+#[serde(rename_all = "camelCase", deny_unknown_fields)]
+pub struct AudioState {
+ pub microphone_peak: u16,
+ pub speaker_peak: u16,
+}
+
/// SDP contains ICE credentials. Bound it at construction and never expose it in diagnostics.
#[derive(Deserialize, PartialEq, Serialize)]
#[serde(try_from = "String")]
@@ -77,6 +85,8 @@ pub enum Message {
DevicesOpened {},
SetAudioControls { controls: AudioControls },
AudioControlsApplied {},
+ InspectAudio {},
+ AudioState { state: AudioState },
Close {},
Closed {},
}
diff --git a/codex-rs/realtime-webrtc/src/session.rs b/codex-rs/realtime-webrtc/src/session.rs
new file mode 100644
index 0000000000..1241a40fa5
--- /dev/null
+++ b/codex-rs/realtime-webrtc/src/session.rs
@@ -0,0 +1,307 @@
+//! Synchronous UI handles for one helper-owning actor. Closing any handle or dropping the last
+//! owner cancels pending I/O; bounded, ordered commands preserve every mute transition.
+//! A shared process-lifetime runtime keeps child reapers alive after an actor stops.
+
+use std::sync::Arc;
+use std::sync::Mutex;
+use std::sync::atomic::AtomicU16;
+use std::sync::atomic::Ordering;
+use std::sync::mpsc as blocking;
+use std::time::Duration;
+
+use anyhow::Result;
+use futures::future::AbortHandle;
+use futures::future::AbortRegistration;
+use futures::future::Abortable;
+use tokio::sync::mpsc;
+
+use crate::AudioControls;
+use crate::SessionDescription;
+use crate::VoiceHost;
+
+// Handshake (5s), runtime initialization (30s), offer gathering (20s), plus overhead.
+const STARTUP_WAIT: Duration = Duration::from_secs(/*secs*/ 60);
+// An in-flight request (5s), answer (20s), devices (5s), controls (5s), plus overhead.
+const ANSWER_WAIT: Duration = Duration::from_secs(/*secs*/ 40);
+static RUNTIME: Mutex<Option<Arc<tokio::runtime::Runtime>>> = Mutex::new(None);
+
+enum Command {
+ Answer(SessionDescription, blocking::SyncSender<()>),
+ Controls(AudioControls),
+}
+
+#[derive(Default)]
+struct State {
+ microphone: AtomicU16,
+ speaker: AtomicU16,
+ error: Mutex<Option<String>>,
+}
+
+struct Owner {
+ sender: mpsc::Sender<Command>,
+ stop: AbortHandle,
+ state: Arc<State>,
+ controls: Arc<Mutex<AudioControls>>,
+}
+
+impl Drop for Owner {
+ fn drop(&mut self) {
+ self.stop.abort();
+ }
+}
+
+/// A cloned handle shares ownership; native media remains entirely inside the child process.
+#[derive(Clone)]
+pub struct RealtimeWebrtcSessionHandle(Arc<Owner>);
+
+impl std::fmt::Debug for RealtimeWebrtcSessionHandle {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ f.write_str("RealtimeWebrtcSessionHandle")
+ }
+}
+
+/// An offer is not proof of connectivity or working devices.
+pub struct StartedRealtimeWebrtcSession {
+ pub offer_sdp: String,
+ pub handle: RealtimeWebrtcSessionHandle,
+}
+
+impl std::fmt::Debug for StartedRealtimeWebrtcSession {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ f.write_str("StartedRealtimeWebrtcSession { offer_sdp: [REDACTED] }")
+ }
+}
+
+pub struct RealtimeWebrtcSession;
+
+impl RealtimeWebrtcSession {
+ /// Check package availability without loading native code or touching any audio device.
+ /// Linux packages may pair a musl application with a GNU helper; startup checks host loading.
+ pub fn is_supported() -> bool {
+ cfg!(any(
+ target_os = "macos",
+ target_os = "linux",
+ all(windows, target_env = "msvc")
+ )) && codex_install_context::InstallContext::current()
+ .package_layout
+ .as_ref()
+ .is_some_and(|package| package_has_runtime(package.package_dir.as_path()))
+ }
+
+ /// Called off the UI thread. Cancellation also owns startup before a handle is returned.
+ pub fn start(abort: AbortRegistration) -> Result<StartedRealtimeWebrtcSession> {
+ let package = codex_install_context::InstallContext::current()
+ .package_layout
+ .clone()
+ .ok_or_else(|| anyhow::anyhow!("voice package unavailable"))?;
+ let build_commit = codex_build_info::BuildInfo::get().build_commit().to_owned();
+ let (sender, receiver) = mpsc::channel(/*buffer*/ 8);
+ let (offer, result) = blocking::sync_channel(/*bound*/ 1);
+ let (stop, stopped) = AbortHandle::new_pair();
+ let state = Arc::new(State::default());
+ let controls = Arc::new(Mutex::new(AudioControls {
+ microphone_muted: false,
+ speaker_suppressed: false,
+ }));
+ let handle = RealtimeWebrtcSessionHandle(Arc::new(Owner {
+ sender,
+ stop,
+ state: state.clone(),
+ controls: controls.clone(),
+ }));
+ let runtime = {
+ let mut runtime = RUNTIME
+ .lock()
+ .unwrap_or_else(std::sync::PoisonError::into_inner);
+ if let Some(runtime) = runtime.as_ref() {
+ runtime.clone()
+ } else {
+ let shared = Arc::new(
+ tokio::runtime::Builder::new_multi_thread()
+ .worker_threads(/*val*/ 1)
+ .enable_all()
+ .build()?,
+ );
+ *runtime = Some(shared.clone());
+ shared
+ }
+ };
+ std::thread::Builder::new()
+ .name("voice-session".into())
+ .spawn(move || {
+ let task = async {
+ let host = VoiceHost::connect(&package, &build_commit)
+ .await?
+ .initialize_runtime()
+ .await?;
+ let (host, sdp) = host.start_transport().await?;
+ offer
+ .send(sdp.into_sdp())
+ .map_err(|_| anyhow::anyhow!("voice startup cancelled"))?;
+ run(host, receiver, &state, &controls).await
+ };
+ let result = runtime.block_on(Abortable::new(Abortable::new(task, abort), stopped));
+ if !matches!(result, Err(_) | Ok(Err(_)) | Ok(Ok(Ok(())))) {
+ *state
+ .error
+ .lock()
+ .unwrap_or_else(std::sync::PoisonError::into_inner) =
+ Some("Voice helper stopped unexpectedly.".into());
+ }
+ })?;
+ let offer_sdp = result
+ .recv_timeout(STARTUP_WAIT)
+ .map_err(|_| anyhow::anyhow!("voice startup failed"))?;
+ Ok(StartedRealtimeWebrtcSession { offer_sdp, handle })
+ }
+}
+
+// Availability is not proof of runtime integrity, device access, or session connectivity.
+fn package_has_runtime(package: &std::path::Path) -> bool {
+ let voice = package.join("codex-resources/voice");
+ let (helper, runtime) = if cfg!(target_os = "macos") {
+ ("bin/codex-voice-host", "lib/libgstreamer-1.0.0.dylib")
+ } else if cfg!(windows) {
+ ("bin/codex-voice-host.exe", "bin/gstreamer-1.0-0.dll")
+ } else {
+ ("bin/codex-voice-host", "lib/libgstreamer-1.0.so.0")
+ };
+ voice.join(helper).is_file() && voice.join(runtime).is_file()
+}
+
+impl RealtimeWebrtcSessionHandle {
+ pub fn close(&self) {
+ self.0.stop.abort();
+ }
+
+ /// Queue an ordered privacy transition; saturation terminates the session instead of dropping it.
+ pub fn set_microphone_muted(&self, muted: bool) -> Result<()> {
+ let mut controls = self
+ .0
+ .controls
+ .lock()
+ .unwrap_or_else(std::sync::PoisonError::into_inner);
+ controls.microphone_muted = muted;
+ self.send(Command::Controls(*controls))
+ }
+
+ pub fn set_speaker_suppressed(&self, suppressed: bool) {
+ let mut controls = self
+ .0
+ .controls
+ .lock()
+ .unwrap_or_else(std::sync::PoisonError::into_inner);
+ controls.speaker_suppressed = suppressed;
+ let _ = self.send(Command::Controls(*controls));
+ }
+
+ /// Called off the UI thread; return only after negotiation, device startup and restored controls.
+ pub fn apply_answer_sdp(&self, answer: String) -> Result<()> {
+ let sdp = SessionDescription::try_from(answer).map_err(anyhow::Error::msg)?;
+ let (complete, result) = blocking::sync_channel(/*bound*/ 1);
+ self.send(Command::Answer(sdp, complete))?;
+ if result.recv_timeout(ANSWER_WAIT).is_err() {
+ self.close();
+ anyhow::bail!("voice connection failed");
+ }
+ Ok(())
+ }
+
+ pub fn take_error(&self) -> Option<String> {
+ self.0
+ .state
+ .error
+ .lock()
+ .unwrap_or_else(std::sync::PoisonError::into_inner)
+ .take()
+ }
+
+ pub fn take_microphone_peak(&self) -> u16 {
+ self.0.state.microphone.swap(/*val*/ 0, Ordering::AcqRel)
+ }
+
+ pub fn take_speaker_peak(&self) -> u16 {
+ self.0.state.speaker.swap(/*val*/ 0, Ordering::AcqRel)
+ }
+
+ fn send(&self, command: Command) -> Result<()> {
+ if self.0.sender.try_send(command).is_err() {
+ *self
+ .0
+ .state
+ .error
+ .lock()
+ .unwrap_or_else(std::sync::PoisonError::into_inner) =
+ Some("Voice control channel unavailable.".into());
+ self.close();
+ anyhow::bail!("voice control channel unavailable");
+ }
+ Ok(())
+ }
+}
+
+async fn run(
+ mut host: VoiceHost,
+ mut commands: mpsc::Receiver<Command>,
+ state: &State,
+ controls: &Mutex<AudioControls>,
+) -> Result<()> {
+ let mut connected = false;
+ let mut poll = tokio::time::interval(Duration::from_millis(/*millis*/ 50));
+ poll.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
+ loop {
+ tokio::select! {
+ biased;
+ command = commands.recv() => match command {
+ Some(Command::Answer(sdp, complete)) if !connected => {
+ host = host.apply_answer(sdp).await?.open_devices().await?;
+ let applied = startup_controls(&mut commands, controls, |initial| {
+ host.begin_audio_controls(initial)
+ })?;
+ applied.await?;
+ connected = true;
+ let _ = complete.send(());
+ }
+ Some(Command::Controls(next)) => {
+ if connected {
+ host.set_audio_controls(next).await?;
+ }
+ }
+ Some(Command::Answer(..)) => anyhow::bail!("voice answer already applied"),
+ None => return host.close().await,
+ },
+ _ = poll.tick() => {
+ let audio = host.inspect_audio().await?;
+ state.microphone.fetch_max(audio.microphone_peak, Ordering::Release);
+ state.speaker.fetch_max(audio.speaker_peak, Ordering::Release);
+ }
+ }
+ }
+}
+
+// Devices are still disabled. The snapshot and request enqueue share the setters' lock;
+// later setters remain ordered after this request without waiting for its acknowledgement.
+fn startup_controls<T>(
+ commands: &mut mpsc::Receiver<Command>,
+ controls: &Mutex<AudioControls>,
+ dispatch: impl FnOnce(AudioControls) -> Result<T>,
+) -> Result<T> {
+ let controls = controls
+ .lock()
+ .unwrap_or_else(std::sync::PoisonError::into_inner);
+ for _ in 0..commands.max_capacity() {
+ match commands.try_recv() {
+ Ok(Command::Controls(_)) => {}
+ Err(mpsc::error::TryRecvError::Empty) => return dispatch(*controls),
+ Ok(Command::Answer(..)) | Err(mpsc::error::TryRecvError::Disconnected) => {
+ anyhow::bail!("voice startup control sequence invalid");
+ }
+ }
+ }
+ anyhow::ensure!(commands.is_empty(), "voice startup controls overloaded");
+ dispatch(*controls)
+}
+
+#[cfg(test)]
+#[path = "session_tests.rs"]
+mod tests;
diff --git a/codex-rs/realtime-webrtc/src/session_tests.rs b/codex-rs/realtime-webrtc/src/session_tests.rs
new file mode 100644
index 0000000000..b3f32e432c
--- /dev/null
+++ b/codex-rs/realtime-webrtc/src/session_tests.rs
@@ -0,0 +1,182 @@
+use super::*;
+use pretty_assertions::assert_eq;
+
+fn handles() -> (
+ RealtimeWebrtcSessionHandle,
+ mpsc::Receiver<Command>,
+ AbortRegistration,
+) {
+ let (sender, receiver) = mpsc::channel(/*buffer*/ 8);
+ let (stop, stopped) = AbortHandle::new_pair();
+ let handle = RealtimeWebrtcSessionHandle(Arc::new(Owner {
+ sender,
+ stop,
+ state: Arc::new(State::default()),
+ controls: Arc::new(Mutex::new(AudioControls {
+ microphone_muted: false,
+ speaker_suppressed: false,
+ })),
+ }));
+ (handle, receiver, stopped)
+}
+
+#[test]
+fn controls_preserve_order_and_independent_settings_and_overflow_closes() {
+ let (handle, mut commands, _) = handles();
+ handle.set_microphone_muted(/*muted*/ true).unwrap();
+ handle.set_speaker_suppressed(/*suppressed*/ true);
+ handle.set_microphone_muted(/*muted*/ false).unwrap();
+ for expected in [(true, false), (true, true), (false, true)] {
+ let Command::Controls(controls) = commands.try_recv().unwrap() else {
+ panic!("wrong command")
+ };
+ assert_eq!(
+ controls,
+ AudioControls {
+ microphone_muted: expected.0,
+ speaker_suppressed: expected.1
+ }
+ );
+ }
+ for _ in 0..8 {
+ handle.set_microphone_muted(/*muted*/ true).unwrap();
+ }
+ assert!(handle.set_microphone_muted(/*muted*/ false).is_err());
+ assert!(handle.0.stop.is_aborted());
+ assert!(handle.take_error().is_some());
+}
+
+#[tokio::test]
+async fn only_final_owner_drop_cancels_and_explicit_close_cancels_all_clones() {
+ let (handle, _commands, stopped) = handles();
+ let clone = handle.clone();
+ drop(handle);
+ assert!(!clone.0.stop.is_aborted());
+ drop(clone);
+ assert!(
+ Abortable::new(std::future::pending::<()>(), stopped)
+ .await
+ .is_err()
+ );
+ let (handle, _commands, stopped) = handles();
+ let clone = handle.clone();
+ handle.close();
+ assert!(clone.0.stop.is_aborted());
+ assert!(
+ Abortable::new(std::future::pending::<()>(), stopped)
+ .await
+ .is_err()
+ );
+}
+
+#[test]
+fn offer_debug_redacts_credentials_and_peaks_are_consumed() {
+ let (handle, _commands, _) = handles();
+ handle.0.state.microphone.store(123, Ordering::Release);
+ handle.0.state.speaker.store(456, Ordering::Release);
+ assert_eq!(
+ (handle.take_microphone_peak(), handle.take_speaker_peak()),
+ (123, 456)
+ );
+ assert_eq!(
+ (handle.take_microphone_peak(), handle.take_speaker_peak()),
+ (0, 0)
+ );
+ let started = StartedRealtimeWebrtcSession {
+ offer_sdp: "synthetic-secret".into(),
+ handle,
+ };
+ assert!(!format!("{started:?}").contains("synthetic-secret"));
+}
+
+#[test]
+fn controls_queued_during_negotiation_precede_first_device_enable() {
+ let (handle, mut commands, _) = handles();
+ // The actor has consumed Answer and is waiting for negotiation/device startup.
+ handle.set_microphone_muted(/*muted*/ true).unwrap();
+ handle.set_speaker_suppressed(/*suppressed*/ true);
+ let controls = startup_controls(&mut commands, &handle.0.controls, Ok).unwrap();
+ assert_eq!(
+ controls,
+ AudioControls {
+ microphone_muted: true,
+ speaker_suppressed: true
+ }
+ );
+ assert!(commands.is_empty());
+ let (complete, _) = blocking::sync_channel(/*bound*/ 1);
+ handle
+ .send(Command::Answer(
+ "synthetic-answer".to_owned().try_into().unwrap(),
+ complete,
+ ))
+ .unwrap();
+ assert!(startup_controls(&mut commands, &handle.0.controls, Ok).is_err());
+}
+
+#[test]
+fn package_availability_requires_helper_and_native_runtime() {
+ let unique = std::time::SystemTime::now()
+ .duration_since(std::time::UNIX_EPOCH)
+ .unwrap()
+ .as_nanos();
+ let root = std::env::temp_dir().join(format!("voice-package-{}-{unique}", std::process::id()));
+ let voice = root.join("codex-resources/voice");
+ std::fs::create_dir_all(voice.join("bin")).unwrap();
+ std::fs::create_dir_all(voice.join("lib")).unwrap();
+ assert!(!package_has_runtime(&root));
+ let helper = voice.join(if cfg!(windows) {
+ "bin/codex-voice-host.exe"
+ } else {
+ "bin/codex-voice-host"
+ });
+ std::fs::write(&helper, b"synthetic helper").unwrap();
+ assert!(!package_has_runtime(&root));
+ let runtime = voice.join(if cfg!(target_os = "macos") {
+ "lib/libgstreamer-1.0.0.dylib"
+ } else if cfg!(windows) {
+ "bin/gstreamer-1.0-0.dll"
+ } else {
+ "lib/libgstreamer-1.0.so.0"
+ });
+ std::fs::write(&runtime, b"synthetic runtime").unwrap();
+ assert!(package_has_runtime(&root));
+ std::fs::remove_file(helper).unwrap();
+ assert!(!package_has_runtime(&root));
+ std::fs::remove_dir_all(root).unwrap();
+}
+
+#[test]
+fn startup_dispatch_is_ordered_but_acknowledgement_does_not_block_setters() {
+ let (handle, mut commands, _) = handles();
+ let applied = startup_controls(&mut commands, &handle.0.controls, |initial| {
+ // This is the request enqueue boundary. A setter cannot commit a newer
+ // snapshot until the initial request is already ordered before it.
+ assert!(matches!(
+ handle.0.controls.try_lock(),
+ Err(std::sync::TryLockError::WouldBlock)
+ ));
+ assert_eq!(
+ initial,
+ AudioControls {
+ microphone_muted: false,
+ speaker_suppressed: false,
+ }
+ );
+ Ok(std::future::pending::<()>())
+ })
+ .unwrap();
+ // The acknowledgement future has not completed or even been polled.
+ handle.set_microphone_muted(/*muted*/ true).unwrap();
+ let Command::Controls(next) = commands.try_recv().unwrap() else {
+ panic!("expected queued mute")
+ };
+ assert_eq!(
+ next,
+ AudioControls {
+ microphone_muted: true,
+ speaker_suppressed: false,
+ }
+ );
+ drop(applied);
+}
diff --git a/codex-rs/realtime-webrtc/tests/common/mod.rs b/codex-rs/realtime-webrtc/tests/common/mod.rs
new file mode 100644
index 0000000000..4996a90777
--- /dev/null
+++ b/codex-rs/realtime-webrtc/tests/common/mod.rs
@@ -0,0 +1,219 @@
+//! Copies this test executable into a private package and dispatches its fake helper before libtest.
+//! The fixture validates protocol order; it never loads native libraries or opens audio devices.
+
+use anyhow::Context;
+use anyhow::Result;
+use anyhow::ensure;
+use codex_realtime_webrtc::AudioState;
+use codex_realtime_webrtc::Message;
+use codex_realtime_webrtc::encode_frame;
+use codex_realtime_webrtc::read_message;
+use std::fs;
+use std::io::Write;
+use std::path::PathBuf;
+use std::process::Command;
+use std::process::Stdio;
+use std::thread;
+use std::time::Duration;
+use std::time::Instant;
+use std::time::SystemTime;
+use std::time::UNIX_EPOCH;
+
+pub const BUILD_COMMIT: &str = "0123456789abcdef0123456789abcdef01234567";
+pub const WAIT: Duration = Duration::from_secs(/*secs*/ 10);
+const APP: &str = if cfg!(windows) { "codex.exe" } else { "codex" };
+const HELPER: &str = if cfg!(windows) {
+ "codex-voice-host.exe"
+} else {
+ "codex-voice-host"
+};
+
+pub fn wait_for(mut ready: impl FnMut() -> bool) -> Result<()> {
+ let deadline = Instant::now() + WAIT;
+ while !ready() {
+ ensure!(Instant::now() < deadline, "fixture deadline expired");
+ thread::sleep(Duration::from_millis(/*millis*/ 5));
+ }
+ Ok(())
+}
+
+#[cfg(unix)]
+pub fn wait_for_helper_reaped(root: &std::path::Path) -> Result<()> {
+ let pid: libc::pid_t = fs::read_to_string(root.join("helper-pid"))?.parse()?;
+ wait_for(|| {
+ // SAFETY: Signal zero checks process existence without delivering a signal.
+ let result = unsafe {
+ libc::kill(pid, /*sig*/ 0)
+ };
+ result == -1 && std::io::Error::last_os_error().raw_os_error() == Some(libc::ESRCH)
+ })
+}
+
+struct Package(PathBuf);
+impl Drop for Package {
+ fn drop(&mut self) {
+ let _ = fs::remove_dir_all(&self.0);
+ }
+}
+
+// The child discovers its installation through current_exe, exactly as the public facade does.
+pub fn package(test: &str) -> Result<Option<PathBuf>> {
+ let source = std::env::current_exe()?;
+ if source.file_name().is_some_and(|name| name == APP) {
+ codex_build_info::BuildInfo::initialize(BUILD_COMMIT);
+ return Ok(Some(
+ source
+ .parent()
+ .context("bin")?
+ .parent()
+ .context("package")?
+ .to_owned(),
+ ));
+ }
+ let nonce = SystemTime::now().duration_since(UNIX_EPOCH)?.as_nanos();
+ let package =
+ Package(std::env::temp_dir().join(format!("voice-actor-{}-{nonce}", std::process::id())));
+ let root = &package.0;
+ fs::create_dir_all(root.join("bin"))?;
+ fs::create_dir_all(root.join("codex-resources/voice/bin"))?;
+ fs::write(root.join("codex-package.json"), "{}")?;
+ fs::copy(&source, root.join("bin").join(APP))?;
+ fs::copy(&source, root.join("codex-resources/voice/bin").join(HELPER))?;
+ let mut child = Command::new(root.join("bin").join(APP))
+ .args(["--exact", test, "--nocapture"])
+ .current_dir(root)
+ .stdout(Stdio::piped())
+ .stderr(Stdio::piped())
+ .spawn()?;
+ let deadline = Instant::now() + WAIT * 3;
+ while child.try_wait()?.is_none() {
+ if Instant::now() >= deadline {
+ child.kill()?;
+ child.wait()?;
+ anyhow::bail!("packaged test timed out");
+ }
+ thread::sleep(Duration::from_millis(/*millis*/ 10));
+ }
+ let output = child.wait_with_output()?;
+ ensure!(
+ output.status.success(),
+ "packaged test failed: {}\n{}\nfixture: {}",
+ String::from_utf8_lossy(&output.stdout),
+ String::from_utf8_lossy(&output.stderr),
+ fs::read_to_string(root.join("fixture-error")).unwrap_or_default()
+ );
+ Ok(None)
+}
+
+#[ctor::ctor]
+fn dispatch_helper() {
+ if std::env::current_exe()
+ .ok()
+ .and_then(|path| path.file_name().map(std::ffi::OsStr::to_owned))
+ .is_some_and(|name| name == HELPER)
+ {
+ if let Err(error) = helper() {
+ let _ = fs::write("fixture-error", format!("{error:#}"));
+ std::process::exit(1);
+ }
+ std::process::exit(0);
+ }
+}
+
+#[derive(PartialEq)]
+enum Stage {
+ Hello,
+ Initialize,
+ Offer,
+ Answer,
+ Devices,
+ Controls,
+ Running,
+}
+
+fn helper() -> Result<()> {
+ let root = std::env::current_dir()?;
+ fs::write(root.join("helper-pid"), std::process::id().to_string())?;
+ let mut input = std::io::stdin().lock();
+ let mut output = std::io::stdout().lock();
+ let mut stage = Stage::Hello;
+ let mut controls_history = Vec::new();
+ while let Some(request) = read_message(&mut input)? {
+ let response = match request {
+ Message::Hello {
+ protocol: 1,
+ build_commit,
+ } if stage == Stage::Hello => {
+ ensure!(build_commit == BUILD_COMMIT, "wrong executable build stamp");
+ stage = Stage::Initialize;
+ Message::Ready {}
+ }
+ Message::InitializeRuntime {} if stage == Stage::Initialize => {
+ fs::write(root.join("initializing"), [])?;
+ if root.join("hold-initialization").exists() {
+ wait_for(|| root.join("release").exists())?;
+ }
+ stage = Stage::Offer;
+ Message::RuntimeReady {}
+ }
+ Message::StartTransport {} if stage == Stage::Offer => {
+ stage = Stage::Answer;
+ Message::Offer {
+ sdp: "synthetic-offer"
+ .to_owned()
+ .try_into()
+ .map_err(anyhow::Error::msg)?,
+ }
+ }
+ Message::ApplyAnswer { sdp } if stage == Stage::Answer => {
+ ensure!(
+ sdp.into_sdp() == "synthetic-answer",
+ "unexpected fixture answer"
+ );
+ fs::write(root.join("answer"), [])?;
+ wait_for(|| root.join("release").exists())?;
+ stage = Stage::Devices;
+ Message::TransportReady {}
+ }
+ Message::OpenDevices {} if stage == Stage::Devices => {
+ stage = Stage::Controls;
+ Message::DevicesOpened {}
+ }
+ Message::SetAudioControls { controls }
+ if stage == Stage::Controls || stage == Stage::Running =>
+ {
+ controls_history.push(controls);
+ fs::write(
+ root.join("controls"),
+ serde_json::to_vec(&controls_history)?,
+ )?;
+ stage = Stage::Running;
+ Message::AudioControlsApplied {}
+ }
+ Message::InspectAudio {} if stage == Stage::Answer || stage == Stage::Running => {
+ if root.join("exit").exists() {
+ return Ok(());
+ }
+ Message::AudioState {
+ state: if stage == Stage::Running {
+ AudioState {
+ microphone_peak: 123,
+ speaker_peak: 456,
+ }
+ } else {
+ AudioState::default()
+ },
+ }
+ }
+ Message::Close {} => {
+ output.write_all(&encode_frame(&Message::Closed {})?)?;
+ output.flush()?;
+ return Ok(());
+ }
+ _ => anyhow::bail!("unexpected fixture protocol sequence"),
+ };
+ output.write_all(&encode_frame(&response)?)?;
+ output.flush()?;
+ }
+ Ok(())
+}
diff --git a/codex-rs/realtime-webrtc/tests/session_actor.rs b/codex-rs/realtime-webrtc/tests/session_actor.rs
new file mode 100644
index 0000000000..44a8917c1b
--- /dev/null
+++ b/codex-rs/realtime-webrtc/tests/session_actor.rs
@@ -0,0 +1,109 @@
+//! Exercises the public session actor against a separate framed-IPC helper, without native audio.
+
+mod common;
+
+use anyhow::Result;
+use codex_realtime_webrtc::AudioControls;
+use codex_realtime_webrtc::RealtimeWebrtcSession;
+use futures::future::AbortHandle;
+use pretty_assertions::assert_eq;
+use std::fs;
+use std::thread;
+
+#[test]
+fn startup_controls_meters_and_helper_loss() -> Result<()> {
+ let Some(root) = common::package("startup_controls_meters_and_helper_loss")? else {
+ return Ok(());
+ };
+ #[cfg(target_os = "linux")]
+ {
+ assert!(!RealtimeWebrtcSession::is_supported());
+ let libraries = root.join("codex-resources/voice/lib");
+ fs::create_dir_all(&libraries)?;
+ fs::write(libraries.join("libgstreamer-1.0.so.0"), b"fixture runtime")?;
+ assert!(RealtimeWebrtcSession::is_supported());
+ }
+ let (_abort, registration) = AbortHandle::new_pair();
+ let started = RealtimeWebrtcSession::start(registration)?;
+ assert_eq!(started.offer_sdp, "synthetic-offer");
+ let handle = started.handle;
+ handle.set_microphone_muted(/*muted*/ true)?;
+ let answer_handle = handle.clone();
+ let answer = thread::spawn(move || answer_handle.apply_answer_sdp("synthetic-answer".into()));
+ common::wait_for(|| root.join("answer").exists())?;
+ handle.set_speaker_suppressed(/*suppressed*/ true);
+ fs::write(root.join("release"), [])?;
+ answer.join().expect("answer thread")?;
+ let initial: Vec<AudioControls> = serde_json::from_slice(&fs::read(root.join("controls"))?)?;
+ assert_eq!(
+ initial,
+ vec![AudioControls {
+ microphone_muted: true,
+ speaker_suppressed: true
+ }]
+ );
+ handle.set_microphone_muted(/*muted*/ false)?;
+ handle.set_speaker_suppressed(/*suppressed*/ false);
+ let expected = serde_json::to_vec(&vec![
+ AudioControls {
+ microphone_muted: true,
+ speaker_suppressed: true,
+ },
+ AudioControls {
+ microphone_muted: false,
+ speaker_suppressed: true,
+ },
+ AudioControls {
+ microphone_muted: false,
+ speaker_suppressed: false,
+ },
+ ])?;
+ common::wait_for(|| fs::read(root.join("controls")).is_ok_and(|data| data == expected))?;
+ common::wait_for(|| handle.take_microphone_peak() == 123)?;
+ common::wait_for(|| handle.take_speaker_peak() == 456)?;
+ fs::write(root.join("exit"), [])?;
+ let mut error = None;
+ common::wait_for(|| {
+ error = handle.take_error();
+ error.is_some()
+ })?;
+ assert_eq!(error.as_deref(), Some("Voice helper stopped unexpectedly."));
+ assert_eq!(handle.take_error(), None);
+ Ok(())
+}
+
+#[test]
+fn external_cancellation_interrupts_startup() -> Result<()> {
+ let Some(root) = common::package("external_cancellation_interrupts_startup")? else {
+ return Ok(());
+ };
+ fs::write(root.join("hold-initialization"), [])?;
+ let (abort, registration) = AbortHandle::new_pair();
+ let (result, received) = std::sync::mpsc::sync_channel(/*bound*/ 1);
+ let startup = thread::spawn(move || {
+ let _ = result.send(RealtimeWebrtcSession::start(registration));
+ });
+ common::wait_for(|| root.join("initializing").exists())?;
+ abort.abort();
+ assert!(
+ received
+ .recv_timeout(std::time::Duration::from_secs(/*secs*/ 2))?
+ .is_err()
+ );
+ startup.join().expect("startup thread");
+ #[cfg(unix)]
+ common::wait_for_helper_reaped(&root)?;
+ Ok(())
+}
+
+#[cfg(unix)]
+#[test]
+fn last_owner_drop_reaps_helper() -> Result<()> {
+ let Some(root) = common::package("last_owner_drop_reaps_helper")? else {
+ return Ok(());
+ };
+ let (_abort, registration) = AbortHandle::new_pair();
+ let started = RealtimeWebrtcSession::start(registration)?;
+ drop(started);
+ common::wait_for_helper_reaped(&root)
+}
diff --git a/codex-rs/tui/Cargo.toml b/codex-rs/tui/Cargo.toml
index 61a7d4853a..040f44f502 100644
--- a/codex-rs/tui/Cargo.toml
+++ b/codex-rs/tui/Cargo.toml
@@ -22,6 +22,7 @@ doctest = false
workspace = true
[dependencies]
+codex-build-info = { workspace = true }
anyhow = { workspace = true }
axum = { workspace = true, default-features = false, features = ["http1", "tokio"] }
base64 = { workspace = true }
diff --git a/codex-rs/tui/src/main.rs b/codex-rs/tui/src/main.rs
index f6261e9dbc..5a196d614a 100644
--- a/codex-rs/tui/src/main.rs
+++ b/codex-rs/tui/src/main.rs
@@ -19,6 +19,7 @@ struct TopCli {
}
fn main() -> anyhow::Result<()> {
+ codex_build_info::initialize!();
arg0_dispatch_or_else(|arg0_paths: Arg0DispatchPaths| async move {
let top_cli = TopCli::parse();
let mut inner = top_cli.inner;
diff --git a/codex-rs/voice-host/src/device_buffers.rs b/codex-rs/voice-host/src/device_buffers.rs
index 97b15198c8..c6112c0480 100644
--- a/codex-rs/voice-host/src/device_buffers.rs
+++ b/codex-rs/voice-host/src/device_buffers.rs
@@ -1,8 +1,10 @@
+//! Meter reads clear accumulated peaks and surface callback failures.
//! Preallocated callback buffers pack small callbacks into full queue slots without locks.
//! Generation changes, rejected capture buffers and capture gaps discard incomplete frames.
//! Capture/render capacity covers the worker deadline; playback and callback limits stay fixed.
use std::sync::atomic::AtomicBool;
+use std::sync::atomic::AtomicU16;
use std::sync::atomic::AtomicU64;
use std::sync::atomic::Ordering;
use std::time::Duration;
@@ -92,9 +94,21 @@ pub(super) struct Buffers {
pub(super) speaker: AtomicU64,
pub(super) serviced: AtomicBool,
pub(super) failed: AtomicBool,
+ pub(super) microphone_peak: AtomicU16,
+ pub(super) speaker_peak: AtomicU16,
}
impl Buffers {
+ pub(super) fn take_state(&self) -> std::io::Result<codex_realtime_webrtc::AudioState> {
+ if self.failed.load(Ordering::Acquire) {
+ return Err(std::io::Error::other("audio device failed"));
+ }
+ Ok(codex_realtime_webrtc::AudioState {
+ microphone_peak: self.microphone_peak.swap(/*val*/ 0, Ordering::AcqRel),
+ speaker_peak: self.speaker_peak.swap(/*val*/ 0, Ordering::AcqRel),
+ })
+ }
+
pub(super) fn new(input_rate: u32, output_rate: u32) -> Self {
// Rates are validated before opening devices. A pending batch can defer
// capture consumption until its freshness deadline plus one in-flight send.
@@ -113,6 +127,8 @@ impl Buffers {
speaker: AtomicU64::new(/*v*/ 1),
serviced: AtomicBool::new(false),
failed: AtomicBool::new(false),
+ microphone_peak: AtomicU16::new(/*v*/ 0),
+ speaker_peak: AtomicU16::new(/*v*/ 0),
}
}
diff --git a/codex-rs/voice-host/src/devices.rs b/codex-rs/voice-host/src/devices.rs
index ede1283af1..6143271e5c 100644
--- a/codex-rs/voice-host/src/devices.rs
+++ b/codex-rs/voice-host/src/devices.rs
@@ -12,6 +12,7 @@ mod processing;
use std::io;
use std::sync::Arc;
+use std::sync::atomic::AtomicU16;
use std::sync::atomic::Ordering;
use std::time::Duration;
use std::time::Instant;
@@ -60,6 +61,10 @@ macro_rules! stream {
}
impl Devices {
+ pub(super) fn take_state(&self) -> io::Result<codex_realtime_webrtc::AudioState> {
+ self.worker.buffers.take_state()
+ }
+
pub(super) fn open() -> io::Result<Self> {
let host = cpal::default_host();
let input = host
@@ -178,6 +183,13 @@ fn bounded_stream_config(
Ok(config)
}
+fn record_peak(peak: &AtomicU16, sample: f32) {
+ peak.fetch_max(
+ (sample.abs().min(1.0) * f32::from(u16::MAX)) as u16,
+ Ordering::Relaxed,
+ );
+}
+
// CPAL reports recoverable underruns/overruns through the same callback as device loss.
fn handle_stream_error(buffers: &Buffers, error: cpal::Error) {
if error.kind() != cpal::ErrorKind::Xrun {
@@ -247,6 +259,7 @@ where
buffers.failed.store(true, Ordering::Release);
return;
}
+ record_peak(&buffers.microphone_peak, *output);
}
if !capture.push(frame, rate, &buffers.capture) {
buffers.failed.store(true, Ordering::Release);
@@ -320,6 +333,7 @@ fn render_output<T>(
let rendered = T::from_sample(output.playback.next(buffers));
frame.fill(rendered);
*sample = f32::from_sample(rendered);
+ record_peak(&buffers.speaker_peak, *sample);
}
if !output.reference.push(reference, rate, &buffers.rendered) {
buffers.failed.store(true, Ordering::Release);
diff --git a/codex-rs/voice-host/src/devices_tests.rs b/codex-rs/voice-host/src/devices_tests.rs
index 0f9752f40e..a8a868fe04 100644
--- a/codex-rs/voice-host/src/devices_tests.rs
+++ b/codex-rs/voice-host/src/devices_tests.rs
@@ -66,6 +66,7 @@ fn startup_output_waits_for_service_but_later_overflow_still_fails() {
&mut state,
);
assert!(buffers.failed.load(Ordering::Acquire));
+ assert!(buffers.take_state().is_err());
}
#[test]
@@ -257,6 +258,25 @@ fn tiny_output_callbacks_pack_references_without_delaying_rendering() {
);
assert_eq!(output, [0.5; 32]);
}
+ for sample in [-0.75, 0.25] {
+ record_peak(&buffers.microphone_peak, sample);
+ }
+ assert_eq!(
+ buffers.take_state().unwrap(),
+ codex_realtime_webrtc::AudioState {
+ microphone_peak: 49151,
+ speaker_peak: 32767,
+ }
+ );
+ assert_eq!(buffers.take_state().unwrap(), Default::default());
+ record_peak(&buffers.microphone_peak, /*sample*/ 1.5);
+ assert_eq!(
+ buffers.take_state().unwrap(),
+ codex_realtime_webrtc::AudioState {
+ microphone_peak: u16::MAX,
+ speaker_peak: 0,
+ }
+ );
assert_eq!(buffers.rendered.len(), 7);
assert!(!buffers.failed.load(Ordering::Acquire));
}
diff --git a/codex-rs/voice-host/src/devices_unavailable.rs b/codex-rs/voice-host/src/devices_unavailable.rs
index 5a59b6ef3f..24fd94d33a 100644
--- a/codex-rs/voice-host/src/devices_unavailable.rs
+++ b/codex-rs/voice-host/src/devices_unavailable.rs
@@ -5,6 +5,10 @@ use std::io;
pub(super) struct Devices;
impl Devices {
+ pub(super) fn take_state(&self) -> io::Result<codex_realtime_webrtc::AudioState> {
+ Self::open().map(|_| codex_realtime_webrtc::AudioState::default())
+ }
+
pub(super) fn open() -> io::Result<Self> {
Err(io::Error::other(
"audio devices unavailable for this helper target",
diff --git a/codex-rs/voice-host/src/main.rs b/codex-rs/voice-host/src/main.rs
index 1d1b7a3470..58fbda71be 100644
--- a/codex-rs/voice-host/src/main.rs
+++ b/codex-rs/voice-host/src/main.rs
@@ -119,6 +119,18 @@ fn run(
.map_err(|_| mpsc::RecvTimeoutError::Disconnected)
};
let reply = match message {
+ Ok(Message::InspectAudio {}) => {
+ if answered && transport.as_ref().is_none_or(|peer| !*peer.ready.borrow()) {
+ return Err(io::Error::other("voice connection closed"));
+ }
+ Message::AudioState {
+ state: devices
+ .as_ref()
+ .map(devices::Devices::take_state)
+ .transpose()?
+ .unwrap_or_default(),
+ }
+ }
Ok(Message::OpenDevices {}) if devices.is_none() && runtime.is_some() && answered => {
devices = Some(devices::Devices::open()?);
Message::DevicesOpened {}
@@ -176,6 +188,7 @@ fn run(
| Message::OpenDevices {}
| Message::DevicesOpened {}
| Message::AudioControlsApplied {}
+ | Message::AudioState { .. }
| Message::Closed {},
) => return Err(io::Error::other("invalid voice control sequence")),
};
diff --git a/codex-rs/voice-host/src/transport.rs b/codex-rs/voice-host/src/transport.rs
index c6f705efbc..f120a70477 100644
--- a/codex-rs/voice-host/src/transport.rs
+++ b/codex-rs/voice-host/src/transport.rs
@@ -64,7 +64,7 @@ pub(crate) struct Transport {
connection: Arc<dyn PeerConnection>,
gathered: Arc<Notify>,
observer: JoinHandle<()>,
- ready: watch::Receiver<bool>,
+ pub(crate) ready: watch::Receiver<bool>,
}
impl Transport {