Lightweight coding agent that runs in your terminal
Rust
Add bounded incoming Opus RTP handling to the voice host (#43100)
Benjamin Carlsson64b482500d00
diff --git a/codex-rs/voice-host/src/audio_track.rs b/codex-rs/voice-host/src/audio_track.rsindex 2dde7d7599..1d3de94890 100644--- a/codex-rs/voice-host/src/audio_track.rs+++ b/codex-rs/voice-host/src/audio_track.rs@@ -14,6 +14,7 @@ use webrtc::media_stream::MediaStreamTrack;use webrtc::media_stream::track_local::static_sample::TrackLocalStaticSample;use webrtc::peer_connection::MediaEngine;+pub(crate) const OPUS_PAYLOAD_TYPE: u8 = 111;pub(crate) const SEND_TIMEOUT: Duration = Duration::from_millis(/*millis*/ 100);pub(crate) struct EncodedAudio {@@ -42,7 +43,7 @@ impl AudioTrack {.register_codec(RTCRtpCodecParameters {rtp_codec: codec.clone(),- payload_type: 111,+ payload_type: OPUS_PAYLOAD_TYPE,},RtpCodecKind::Audio,)@@ -84,7 +85,7 @@ impl AudioTrack {self.track.write_sample(self.ssrc,- /*payload_type*/ 111,+ /*payload_type*/ OPUS_PAYLOAD_TYPE,&Sample {duration,..Default::default()@@ -99,7 +100,7 @@ impl AudioTrack {self.track.write_sample(self.ssrc,- /*payload_type*/ 111,+ /*payload_type*/ OPUS_PAYLOAD_TYPE,&Sample {data: frame.data.into(),duration,diff --git a/codex-rs/voice-host/src/incoming.rs b/codex-rs/voice-host/src/incoming.rsnew file mode 100644index 0000000000..b230d0eda5--- /dev/null+++ b/codex-rs/voice-host/src/incoming.rs@@ -0,0 +1,169 @@+//! Takes incoming Opus RTP before the upstream track queue, preserving network arrival time.+//! Packet and byte permits follow owned data through the consumer, not merely through this queue.++use std::sync::Arc;+use std::sync::atomic::AtomicBool;+use std::sync::atomic::AtomicU64;+use std::sync::atomic::Ordering;+use std::time::Duration;+use std::time::Instant;++use rtc::interceptor::Interceptor;+use rtc::interceptor::NoopInterceptor;+use rtc::interceptor::Packet;+use rtc::interceptor::StreamInfo;+use rtc::interceptor::TaggedPacket;+use rtc::interceptor::interceptor;+use rtc::sansio;+use rtc::shared::error::Error;+use rtc::shared::marshal::Marshal;+use rtc::shared::marshal::MarshalSize;+use tokio::sync::OwnedSemaphorePermit;+use tokio::sync::Semaphore;+use tokio::sync::mpsc;++const PACKETS: usize = 64;+const BYTES: usize = 2 * 1024 * 1024;+const PACKET_BYTES: usize = 64 * 1024;++struct State {+ epoch: AtomicU64,+ failed: AtomicBool,+ packets: Arc<Semaphore>,+ bytes: Arc<Semaphore>,+}++pub(crate) struct ReceivedRtp {+ data: Vec<u8>,+ pub(crate) at: Instant,+ epoch: u64,+ _packet: OwnedSemaphorePermit,+ _bytes: OwnedSemaphorePermit,+}++impl AsRef<[u8]> for ReceivedRtp {+ fn as_ref(&self) -> &[u8] {+ &self.data+ }+}++pub(crate) struct Incoming {+ state: Arc<State>,+ receiver: mpsc::Receiver<ReceivedRtp>,+}++#[derive(Interceptor)]+pub(crate) struct Ingress {+ #[next]+ next: NoopInterceptor,+ state: Arc<State>,+ sender: mpsc::Sender<ReceivedRtp>,+ stream: Option<u32>,+}++impl Incoming {+ pub(crate) fn new() -> (Self, Ingress) {+ let state = Arc::new(State {+ epoch: AtomicU64::new(/*v*/ 1),+ failed: AtomicBool::new(false),+ packets: Arc::new(Semaphore::new(/*permits*/ PACKETS)),+ bytes: Arc::new(Semaphore::new(/*permits*/ BYTES)),+ });+ let (sender, receiver) = mpsc::channel(/*buffer*/ PACKETS);+ (+ Self {+ state: state.clone(),+ receiver,+ },+ Ingress {+ next: NoopInterceptor::new(),+ state,+ sender,+ stream: None,+ },+ )+ }++ pub(crate) fn set_suppressed(&self, suppressed: bool) -> Result<(), &'static str> {+ let epoch = self.state.epoch.load(Ordering::Acquire);+ if (epoch % 2 == 1) != suppressed {+ self.state.epoch.store(+ epoch.checked_add(1).ok_or("audio epoch exhausted")?,+ Ordering::Release,+ );+ }+ Ok(())+ }++ pub(crate) fn take(&mut self) -> Result<Option<ReceivedRtp>, &'static str> {+ if self.state.failed.load(Ordering::Acquire) {+ return Err("incoming audio failed");+ }+ let epoch = self.state.epoch.load(Ordering::Acquire);+ for _ in 0..PACKETS {+ let Ok(packet) = self.receiver.try_recv() else {+ return Ok(None);+ };+ if epoch.is_multiple_of(2) && packet.epoch == epoch {+ if packet.at.elapsed() > Duration::from_secs(/*secs*/ 1) {+ return Err("incoming audio fell behind");+ }+ return Ok(Some(packet));+ }+ }+ Ok(None)+ }+}++#[interceptor]+impl Ingress {+ #[overrides]+ fn handle_read(&mut self, message: TaggedPacket) -> Result<(), Self::Error> {+ let Packet::Rtp(packet) = message.message else {+ return Ok(());+ };+ let epoch = self.state.epoch.load(Ordering::Acquire);+ if epoch % 2 == 1 || self.state.failed.load(Ordering::Acquire) {+ return Ok(());+ }+ let size = packet.marshal_size();+ let accepted = (|| {+ if size > PACKET_BYTES+ || packet.header.payload_type != crate::audio_track::OPUS_PAYLOAD_TYPE+ || self.stream.is_some_and(|ssrc| ssrc != packet.header.ssrc)+ {+ return None;+ }+ self.stream = Some(packet.header.ssrc);+ let packet_permit = self.state.packets.clone().try_acquire_owned().ok()?;+ let bytes_permit = self+ .state+ .bytes+ .clone()+ .try_acquire_many_owned(size as u32)+ .ok()?;+ let mut data = vec![0; size];+ if packet.marshal_to(&mut data).ok()? != size {+ return None;+ }+ self.sender+ .try_send(ReceivedRtp {+ data,+ at: message.now,+ epoch,+ _packet: packet_permit,+ _bytes: bytes_permit,+ })+ .ok()+ })();+ if accepted.is_none() {+ self.state.failed.store(true, Ordering::Release);+ }+ // Consume here: no second copy is retained by the upstream track-event queue.+ Ok(())+ }+}++#[cfg(test)]+#[path = "incoming_tests.rs"]+mod tests;diff --git a/codex-rs/voice-host/src/incoming_tests.rs b/codex-rs/voice-host/src/incoming_tests.rsnew file mode 100644index 0000000000..bc32d83845--- /dev/null+++ b/codex-rs/voice-host/src/incoming_tests.rs@@ -0,0 +1,100 @@+use super::*;+use pretty_assertions::assert_eq;+use rtc::rtp::header::Header;+use rtc::sansio::Protocol;++fn pair() -> (Incoming, Ingress) {+ let (incoming, ingress) = Incoming::new();+ incoming.set_suppressed(/*suppressed*/ false).unwrap();+ (incoming, ingress)+}++fn packet(size: usize, at: Instant) -> TaggedPacket {+ TaggedPacket {+ now: at,+ transport: Default::default(),+ message: Packet::Rtp(rtc::rtp::Packet {+ header: Header {+ version: 2,+ payload_type: 111,+ ssrc: 7,+ ..Default::default()+ },+ payload: vec![5; size - 12].into(),+ }),+ }+}++#[test]+fn arrival_and_rtp_bytes_survive_the_adapter_without_an_upstream_copy() {+ let (mut incoming, mut ingress) = pair();+ let at = Instant::now() - Duration::from_millis(/*millis*/ 20);+ let input = packet(/*size*/ 32, at);+ let Packet::Rtp(expected) = &input.message else {+ panic!()+ };+ let expected = expected.marshal().unwrap();+ ingress.handle_read(input).unwrap();+ let received = incoming.take().unwrap().unwrap();+ assert_eq!((received.at, received.as_ref()), (at, expected.as_ref()));+ assert!(ingress.poll_read().is_none());+}++#[test]+fn consumed_packets_keep_their_budget_until_the_last_owner_drops_them() {+ for size in [32, PACKET_BYTES] {+ let (mut incoming, mut ingress) = pair();+ let count = PACKETS.min(BYTES / size);+ let mut held = Vec::new();+ for _ in 0..count {+ ingress.handle_read(packet(size, Instant::now())).unwrap();+ held.push(incoming.take().unwrap().unwrap());+ }+ drop(held.pop());+ ingress.handle_read(packet(size, Instant::now())).unwrap();+ held.push(incoming.take().unwrap().unwrap());+ ingress.handle_read(packet(size, Instant::now())).unwrap();+ assert_eq!(incoming.take().err(), Some("incoming audio failed"));+ drop(held);+ assert_eq!(+ (+ incoming.state.packets.available_permits(),+ incoming.state.bytes.available_permits()+ ),+ (PACKETS, BYTES)+ );+ }+}++#[test]+fn suppression_discards_queued_and_in_flight_old_epochs() {+ let (mut incoming, mut ingress) = pair();+ ingress+ .handle_read(packet(/*size*/ 32, Instant::now()))+ .unwrap();+ incoming.set_suppressed(/*suppressed*/ true).unwrap();+ ingress+ .handle_read(packet(/*size*/ 32, Instant::now()))+ .unwrap();+ incoming.set_suppressed(/*suppressed*/ false).unwrap();+ assert!(incoming.take().unwrap().is_none());+ ingress+ .handle_read(packet(/*size*/ 32, Instant::now()))+ .unwrap();+ assert!(incoming.take().unwrap().is_some());+}++#[test]+fn invalid_stream_and_oversize_packets_fail_without_returning_media() {+ for size in [32, PACKET_BYTES + 1] {+ let (mut incoming, mut ingress) = pair();+ let mut input = packet(size, Instant::now());+ if size == 32+ && let Packet::Rtp(packet) = &mut input.message+ {+ packet.header.payload_type = 96;+ }+ ingress.handle_read(input).unwrap();+ assert_eq!(incoming.take().err(), Some("incoming audio failed"));+ }+}diff --git a/codex-rs/voice-host/src/main.rs b/codex-rs/voice-host/src/main.rsindex 58fbda71be..580de49418 100644--- a/codex-rs/voice-host/src/main.rs+++ b/codex-rs/voice-host/src/main.rs@@ -20,6 +20,7 @@ mod audio_track;path = "devices_unavailable.rs")]mod devices;+mod incoming;mod runtime;mod transport;mod transport_runtime;@@ -104,6 +105,10 @@ fn run(let mut answered = false;let mut devices: Option<devices::Devices> = None;loop {+ if let Some(peer) = &mut transport {+ // The next media stage feeds these owned packets into the jitter/decode pipeline.+ while peer.incoming.take().map_err(io::Error::other)?.is_some() {}+ }let message = if let Some(devices) = &mut devices {let peer = transport.as_mut()@@ -136,6 +141,12 @@ fn run(Message::DevicesOpened {}}Ok(Message::SetAudioControls { controls }) => {+ transport+ .as_ref()+ .ok_or_else(|| io::Error::other("voice peer not started"))?+ .incoming+ .set_suppressed(controls.speaker_suppressed)+ .map_err(io::Error::other)?;devices.as_mut().ok_or_else(|| io::Error::other("audio devices not open"))?diff --git a/codex-rs/voice-host/src/transport.rs b/codex-rs/voice-host/src/transport.rsindex f120a70477..3b0646576d 100644--- a/codex-rs/voice-host/src/transport.rs+++ b/codex-rs/voice-host/src/transport.rs@@ -60,6 +60,7 @@ impl PeerConnectionEventHandler for Events {}pub(crate) struct Transport {+ pub(crate) incoming: crate::incoming::Incoming,pub(crate) audio: crate::audio_track::AudioTrack,connection: Arc<dyn PeerConnection>,gathered: Arc<Notify>,@@ -74,6 +75,7 @@ impl Transport {async fn with_runtime(runtime: Arc<dyn webrtc::runtime::Runtime>) -> Result<Self> {let (media, audio) = crate::audio_track::AudioTrack::new()?;+ let (incoming, ingress) = crate::incoming::Incoming::new();let gathered = Arc::new(Notify::new());// Upstream defaults exhaust checks after 1.4s, including checks sent before// a TCP connection exists. Keep probing throughout our negotiation deadline.@@ -84,6 +86,7 @@ impl Transport {settings.set_ice_connection_attempts(Some(check_interval), Some(attempts));let connection: Arc<dyn PeerConnection> = Arc::new(PeerConnectionBuilder::new()+ .with_interceptor_registry(rtc::interceptor::Registry::from(ingress)).with_media_engine(media).with_runtime(runtime).with_setting_engine(settings)@@ -128,6 +131,7 @@ impl Transport {sender.send_replace(false);});Ok(Self {+ incoming,audio,connection,gathered,diff --git a/codex-rs/voice-host/src/transport_tests.rs b/codex-rs/voice-host/src/transport_tests.rsindex 9c9f338b71..20eda40ad9 100644--- a/codex-rs/voice-host/src/transport_tests.rs+++ b/codex-rs/voice-host/src/transport_tests.rs@@ -55,7 +55,7 @@ async fn check_negotiation(runtime: Arc<dyn webrtc::runtime::Runtime>) {let gathered = Arc::new(Notify::new());let mut settings = webrtc::peer_connection::SettingEngine::default();settings.set_lite(/*lite*/ true);- let (media, _) = crate::audio_track::AudioTrack::new().unwrap();+ let (media, mut remote_audio) = crate::audio_track::AudioTrack::new().unwrap();let builder = PeerConnectionBuilder::new().with_media_engine(media).with_setting_engine(settings)@@ -68,6 +68,7 @@ async fn check_negotiation(runtime: Arc<dyn webrtc::runtime::Runtime>) {.build().await.unwrap();+ remote.add_track(remote_audio.track.clone()).await.unwrap();let mut local = Transport::with_runtime(runtime.clone()).await.unwrap();let offer = RTCSessionDescription::offer(local.offer().await.unwrap()).unwrap();remote.set_remote_description(offer).await.unwrap();@@ -118,6 +119,26 @@ async fn check_negotiation(runtime: Arc<dyn webrtc::runtime::Runtime>) {(RTCDataChannelState::Open, true));channel.send_text("synthetic sideband event").await.unwrap();+ local.incoming.set_suppressed(/*suppressed*/ false).unwrap();+ let sent = std::time::Instant::now();+ remote_audio+ .send(crate::audio_track::EncodedAudio {+ data: vec![0xf8, 0xff, 0xfe],+ at: sent,+ })+ .await+ .unwrap();+ let received = loop {+ if let Some(packet) = local.incoming.take().unwrap() {+ break packet;+ }+ tokio::time::sleep(Duration::from_millis(/*millis*/ 5)).await;+ };+ assert!(received.at >= sent);+ assert_eq!(+ &received.as_ref()[received.as_ref().len() - 3..],+ &[0xf8, 0xff, 0xfe]+ );local.close().await.unwrap();assert!(local.ready.has_changed().is_err());remote.close().await.unwrap();