Skip to content

armgit / codex

Public Imported from GitHub
0

Lightweight coding agent that runs in your terminal

Rust

Accept voice response audio before captions on quiet turns (#44925)

Benjamin Carlsson1b5e27c7f0e9
diff --git a/codex-rs/tui/src/chatwidget/realtime.rs b/codex-rs/tui/src/chatwidget/realtime.rs
index 9f0a48cbb8..c7330d40d4 100644
--- a/codex-rs/tui/src/chatwidget/realtime.rs
+++ b/codex-rs/tui/src/chatwidget/realtime.rs
@@ -1201,7 +1201,7 @@ impl ChatWidget {
self.realtime_conversation.interruption_acknowledged_until =
Some(Instant::now() + INTERRUPTION_ACKNOWLEDGMENT);
}
- self.suppress_realtime_speaker();
+ self.suppress_active_realtime_speaker();
}
if active
&& role == "assistant"
@@ -1454,7 +1454,7 @@ impl ChatWidget {
.input_generation
.wrapping_add(/*rhs*/ 1);
self.realtime_conversation.latest_input_was_voice = true;
- self.suppress_realtime_speaker();
+ self.suppress_active_realtime_speaker();
}
}
}
diff --git a/codex-rs/tui/src/chatwidget/realtime/recording_controls.rs b/codex-rs/tui/src/chatwidget/realtime/recording_controls.rs
index e42c7525d5..baccdeed88 100644
--- a/codex-rs/tui/src/chatwidget/realtime/recording_controls.rs
+++ b/codex-rs/tui/src/chatwidget/realtime/recording_controls.rs
@@ -76,6 +76,39 @@ impl ChatWidget {
}
}
+ pub(super) fn suppress_active_realtime_speaker(&mut self) {
+ // Quiet turns must not wait for captions before accepting their first
+ // audio packets. Only interrupt output that may belong to an older turn.
+ if self
+ .realtime_conversation
+ .assistant_transcript_generation
+ .is_some()
+ || self
+ .realtime_conversation
+ .pending_speech
+ .iter()
+ .any(|pending| {
+ pending.input_generation != self.realtime_conversation.input_generation
+ && !pending.captioned
+ && matches!(
+ pending.state,
+ PendingSpeechState::Queued(_) | PendingSpeechState::Accepted
+ )
+ })
+ || self.realtime_conversation.speaker_level > 0
+ || self
+ .realtime_conversation
+ .speaker_active_until
+ .is_some_and(|deadline| deadline > Instant::now())
+ || self
+ .realtime_conversation
+ .speaker_suppression_generation
+ .is_some()
+ {
+ self.suppress_realtime_speaker();
+ }
+ }
+
pub(super) fn suppress_realtime_speaker(&mut self) {
self.realtime_conversation.speaker_suppression_generation =
Some(self.realtime_conversation.input_generation);
diff --git a/codex-rs/tui/src/chatwidget/realtime_tests/handoffs.rs b/codex-rs/tui/src/chatwidget/realtime_tests/handoffs.rs
index adb10ee428..fa3e991d14 100644
--- a/codex-rs/tui/src/chatwidget/realtime_tests/handoffs.rs
+++ b/codex-rs/tui/src/chatwidget/realtime_tests/handoffs.rs
@@ -416,7 +416,7 @@ async fn newer_voice_delegation_on_the_same_turn_speaks_only_its_own_answer() {
if text.as_str() == "answer to the newer question"
));
let owner = chat.realtime_conversation.speaker_suppression_generation;
- assert_eq!(owner, Some(chat.realtime_conversation.input_generation));
+ assert_eq!(owner, None);
chat.on_realtime_transcript_delta("assistant".to_string(), "answer".to_string());
let owner = chat.realtime_conversation.speaker_suppression_generation;
assert!(owner.is_none());
diff --git a/codex-rs/tui/src/chatwidget/realtime_tests/transcripts.rs b/codex-rs/tui/src/chatwidget/realtime_tests/transcripts.rs
index 7388099886..a7c28b0f97 100644
--- a/codex-rs/tui/src/chatwidget/realtime_tests/transcripts.rs
+++ b/codex-rs/tui/src/chatwidget/realtime_tests/transcripts.rs
@@ -122,6 +122,7 @@ async fn completed_user_transcript_keeps_the_old_assistant_turn_suppressed() {
async fn final_only_assistant_transcript_cannot_release_an_interruption() {
let (mut chat, _sender, _events, _ops) = make_chatwidget_manual_with_sender().await;
activate_voice(&mut chat);
+ chat.realtime_conversation.speaker_level = 1;
chat.on_realtime_transcript_delta("user".into(), "Stop".into());
let generation = chat.realtime_conversation.input_generation;
chat.on_realtime_transcript_done("user".into(), "Stop".into());
@@ -153,7 +154,7 @@ async fn transcript_deltas_track_the_current_speaker() {
chat.on_realtime_transcript_done("assistant".to_string(), "Old".to_string());
assert_eq!(
chat.realtime_conversation.speaker_suppression_generation,
- Some(chat.realtime_conversation.input_generation)
+ None
);
chat.on_realtime_transcript_done("user".to_string(), "Hello there".to_string());
@@ -163,9 +164,9 @@ async fn transcript_deltas_track_the_current_speaker() {
chat.on_realtime_transcript_delta("user".to_string(), "Again".to_string());
chat.on_realtime_transcript_done("user".to_string(), "Again".to_string());
- assert!(owner(&chat).is_some());
+ assert!(owner(&chat).is_none());
chat.on_realtime_transcript_delta("assistant".to_string(), "".to_string());
- assert!(owner(&chat).is_some());
+ assert!(owner(&chat).is_none());
chat.on_realtime_transcript_delta("assistant".to_string(), "Hi".to_string());
assert!(owner(&chat).is_none());
@@ -772,3 +773,57 @@ async fn spoken_user_transcript_preserves_red_chevron_and_canonical_history() {
.any(|line| line.to_string() == "› hello world")
);
}
+
+#[tokio::test]
+async fn quiet_voice_turns_accept_audio_before_assistant_captions() {
+ let (mut chat, _sender, _events, _ops) = make_chatwidget_manual_with_sender().await;
+ activate_voice(&mut chat);
+ for user_delta in [Some("First question"), None, Some("Third question")] {
+ if let Some(delta) = user_delta {
+ chat.on_realtime_transcript_delta("user".into(), delta.into());
+ assert_eq!(
+ chat.realtime_conversation.speaker_suppression_generation,
+ None
+ );
+ }
+ chat.on_realtime_transcript_done("user".into(), "Question".into());
+ // Audio can arrive before captions: keep the helper accepting it now.
+ assert_eq!(
+ chat.realtime_conversation.speaker_suppression_generation,
+ None
+ );
+ chat.on_realtime_transcript_delta("assistant".into(), "Answer".into());
+ chat.on_realtime_transcript_done("assistant".into(), "Answer".into());
+ }
+}
+
+#[tokio::test]
+async fn new_voice_turn_interrupts_uncaptioned_queued_speech() {
+ use super::super::PendingRealtimeSpeech;
+ use super::super::PendingSpeechState;
+
+ for (state, captioned, expected) in [
+ (PendingSpeechState::Queued(1), false, Some(1)),
+ (PendingSpeechState::Accepted, false, Some(1)),
+ (PendingSpeechState::AwaitingTurn, false, None),
+ (PendingSpeechState::Accepted, true, None),
+ ] {
+ let (mut chat, _sender, _events, _ops) = make_chatwidget_manual_with_sender().await;
+ let thread_id = activate_voice(&mut chat);
+ chat.realtime_conversation
+ .pending_speech
+ .push_back(PendingRealtimeSpeech {
+ state,
+ captioned,
+ input_generation: 0,
+ thread_id,
+ turn_id: "old-turn".into(),
+ item: agent_item("answer", "Old answer", Some(MessagePhase::FinalAnswer)),
+ });
+ chat.on_realtime_transcript_delta("user".into(), "New question".into());
+ assert_eq!(
+ chat.realtime_conversation.speaker_suppression_generation,
+ expected
+ );
+ }
+}
diff --git a/codex-rs/tui/src/chatwidget/tests/status_and_layout.rs b/codex-rs/tui/src/chatwidget/tests/status_and_layout.rs
index daedcbc21b..3025dca32e 100644
--- a/codex-rs/tui/src/chatwidget/tests/status_and_layout.rs
+++ b/codex-rs/tui/src/chatwidget/tests/status_and_layout.rs
@@ -41,6 +41,7 @@ async fn voice_live_transcript_renders_beside_the_streamed_cell() {
let (mut chat, _rx, _ops) = make_chatwidget_manual(/*model_override*/ None).await;
chat.local_settings.tui.animations = false;
activate_voice_for_thread(&mut chat, ThreadId::new());
+ chat.update_realtime_footer();
chat.transcript.active_cell = Some(Box::new(history_cell::StreamingAgentTailCell::new(
vec![Line::from("Agent answer arriving").into()],
/*is_first_line*/ true,