Lightweight coding agent that runs in your terminal
Rust
Animate live voice transcripts with split-flap tiles (#43656)
Benjamin Carlsson4e93cf9b4e4e
diff --git a/codex-rs/tui/src/chatwidget.rs b/codex-rs/tui/src/chatwidget.rsindex cb8ff353e0..0ccb35e6e6 100644--- a/codex-rs/tui/src/chatwidget.rs+++ b/codex-rs/tui/src/chatwidget.rs@@ -1972,7 +1972,9 @@ impl ChatWidget {is_stream_continuation: cell.map(|cell| cell.is_stream_continuation()).unwrap_or(false),- animation_tick: cell.and_then(|cell| cell.transcript_animation_tick()),+ animation_tick: cell+ .and_then(|cell| cell.transcript_animation_tick())+ .or_else(|| realtime_cell.and_then(|cell| cell.transcript_animation_tick())),})}diff --git a/codex-rs/tui/src/chatwidget/realtime.rs b/codex-rs/tui/src/chatwidget/realtime.rsindex 48beb9ec90..0c0950ed32 100644--- a/codex-rs/tui/src/chatwidget/realtime.rs+++ b/codex-rs/tui/src/chatwidget/realtime.rs@@ -6,11 +6,13 @@ mod recording_controls;use super::ChatWidget;use super::HistoryCell;use super::PARENT_OWNED_INPUT_MESSAGE;+use super::realtime_split_flap::SplitFlapTranscriptCell;use super::realtime_split_flap::VoiceAmplitudeHistory;use crate::app_command::AppCommand;use crate::app_event::AppEvent;use crate::history_cell;use crate::key_hint;+use crate::motion::MotionMode;use codex_app_server_protocol::ThreadItem;use codex_app_server_protocol::UserInput;use codex_features::Feature;@@ -1244,9 +1246,22 @@ impl ChatWidget {.transcript_role.as_deref().unwrap_or("");- self.realtime_conversation.live_transcript_cell = Some(- self.realtime_transcript_history_cell(role, &self.realtime_conversation.transcript),+ let previous = self+ .realtime_conversation+ .live_transcript_cell+ .as_deref()+ .and_then(|cell| cell.as_any().downcast_ref::<SplitFlapTranscriptCell>());+ let cell =+ self.realtime_transcript_history_cell(role, &self.realtime_conversation.transcript);+ let live_cell = SplitFlapTranscriptCell::new(+ cell,+ role,+ &self.realtime_conversation.transcript,+ previous,+ MotionMode::from_animations_enabled(self.config.animations),+ self.frame_requester.clone(),);+ self.realtime_conversation.live_transcript_cell = Some(Box::new(live_cell));self.bump_active_cell_revision();self.request_redraw();}diff --git a/codex-rs/tui/src/chatwidget/realtime_split_flap.rs b/codex-rs/tui/src/chatwidget/realtime_split_flap.rsindex cff8ef47a9..2adf54120c 100644--- a/codex-rs/tui/src/chatwidget/realtime_split_flap.rs+++ b/codex-rs/tui/src/chatwidget/realtime_split_flap.rs@@ -1,9 +1,28 @@-//! Bounded real amplitude histories for voice recording controls.+//! A bounded, presentation-only split-flap board for live voice transcripts.++use super::HistoryCell;+use crate::motion::MotionMode;+use crate::terminal_hyperlinks::HyperlinkLine;+use crate::terminal_hyperlinks::visible_lines;+use crate::tui::FrameRequester;+use ratatui::style::Color;+use ratatui::style::Style;+use ratatui::text::Line;+use ratatui::text::Span;+use std::time::Duration;+use std::time::Instant;+use unicode_segmentation::UnicodeSegmentation;#[cfg(test)]#[path = "realtime_split_flap_tests.rs"]mod tests;+const FRAME_INTERVAL: Duration = Duration::from_millis(45);+const ANIMATION_DURATION: Duration = Duration::from_millis(675);+const TILE_SETTLE_DURATION: Duration = Duration::from_millis(180);+const TILE_AFTERGLOW_DURATION: Duration = Duration::from_millis(135);+const FLAP_SAMPLE_WINDOW: usize = 24;+/// Four bounded, calibrated amplitude samples; this is not frequency analysis.#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]pub(super) struct VoiceAmplitudeHistory([usize; 4]);@@ -31,3 +50,280 @@ impl VoiceAmplitudeHistory {self.0.iter().copied().max().unwrap_or_default()}}++#[derive(Debug)]+pub(super) struct SplitFlapTranscriptCell {+ inner: Box<dyn HistoryCell>,+ role: String,+ target: String,+ visible_target: String,+ flap_sample: Vec<u8>,+ tile_arrivals: Vec<Instant>,+ started_at: Instant,+ phase_started_at: Instant,+ motion_mode: MotionMode,+ frame_requester: FrameRequester,+}++impl SplitFlapTranscriptCell {+ pub(super) fn new(+ inner: Box<dyn HistoryCell>,+ role: &str,+ target: &str,+ previous: Option<&Self>,+ motion_mode: MotionMode,+ frame_requester: FrameRequester,+ ) -> Self {+ let now = Instant::now();+ let visible_target = visible_glyphs(inner.as_ref());+ let mut flap_sample = visible_target+ .bytes()+ .rev()+ .filter(u8::is_ascii_alphabetic)+ .take(FLAP_SAMPLE_WINDOW)+ .collect::<Vec<_>>();+ if flap_sample.is_empty() {+ flap_sample.extend(visible_target.bytes().rev().take(FLAP_SAMPLE_WINDOW));+ }+ flap_sample.reverse();+ let previous = previous.filter(|cell| {+ cell.role == role+ && target.starts_with(&cell.target)+ && visible_target.starts_with(&cell.visible_target)+ });+ let mut tile_arrivals = previous+ .map(|cell| cell.tile_arrivals.clone())+ .unwrap_or_default();+ let existing_tiles = tile_arrivals.len();+ for index in existing_tiles..visible_target.len() {+ let offset = u64::try_from(index.saturating_sub(existing_tiles))+ .unwrap_or(u64::MAX)+ .saturating_mul(/*rhs*/ 8)+ .min(/*other*/ 56);+ let start = if previous.is_some() {+ now.checked_sub(FRAME_INTERVAL).unwrap_or(now)+ } else {+ now+ };+ tile_arrivals.push(start + Duration::from_millis(offset));+ }+ let cell = Self {+ inner,+ role: role.to_string(),+ target: target.to_string(),+ visible_target,+ flap_sample,+ tile_arrivals,+ started_at: now,+ phase_started_at: previous.map(|cell| cell.phase_started_at).unwrap_or(now),+ motion_mode,+ frame_requester,+ };+ if cell.is_animating(Duration::ZERO) {+ cell.frame_requester.schedule_frame_in(FRAME_INTERVAL);+ }+ cell+ }++ fn is_animating(&self, elapsed: Duration) -> bool {+ self.motion_mode == MotionMode::Animated+ && elapsed < ANIMATION_DURATION+ && !self.tile_arrivals.is_empty()+ }++ fn animate_lines(+ &self,+ mut lines: Vec<HyperlinkLine>,+ width: u16,+ elapsed: Duration,+ ) -> Vec<HyperlinkLine> {+ if self.motion_mode == MotionMode::Reduced {+ return lines;+ }++ let board_style = Style::new().bg(Color::Black).fg(Color::Gray);+ let mut glyph_index = 0;+ let mut word_is_unsettled = false;+ let now = self.started_at + elapsed;+ let phase_elapsed = now.saturating_duration_since(self.phase_started_at);+ let final_line = lines.iter().rposition(|line| {+ line.line+ .spans+ .iter()+ .any(|span| !span.content.trim().is_empty())+ });+ for (line_index, line) in lines.iter_mut().enumerate() {+ if line+ .line+ .spans+ .iter()+ .all(|span| span.content.trim().is_empty())+ {+ continue;+ }++ let original_spans = std::mem::take(&mut line.line.spans);+ line.line.style = line.line.style.patch(board_style);+ for span in original_spans {+ for grapheme in span.content.graphemes(/*is_extended*/ true) {+ let mut style = span.style.patch(board_style);+ let text = if is_flippable(grapheme) {+ let position = glyph_index;+ glyph_index += 1;+ let arrival = self+ .tile_arrivals+ .get(position)+ .copied()+ .unwrap_or(self.started_at);+ let tile_elapsed = now.saturating_duration_since(arrival);+ let (text, flipping) = flap_glyph(+ grapheme,+ position,+ tile_elapsed,+ phase_elapsed,+ &self.flap_sample,+ );+ if flipping {+ word_is_unsettled = true;+ style = style.fg(Color::DarkGray);+ } else if tile_elapsed < TILE_SETTLE_DURATION + TILE_AFTERGLOW_DURATION {+ style = style.fg(if self.role == "user" {+ Color::Cyan+ } else {+ Color::Magenta+ });+ }+ text+ } else {+ if grapheme == "›" {+ style = style.fg(Color::Cyan);+ } else if grapheme == "•" {+ style = style.fg(Color::Magenta);+ }+ if grapheme.chars().all(char::is_whitespace) {+ word_is_unsettled = false;+ }+ if word_is_unsettled && is_punctuation(grapheme) {+ " ".to_string()+ } else {+ grapheme.to_string()+ }+ };+ line.line.spans.push(Span::styled(text, style));+ }+ }++ let remaining = usize::from(width).saturating_sub(line.width());+ if Some(line_index) == final_line+ && remaining > 1+ && self.is_animating(elapsed)+ && self.tile_arrivals.first().is_some_and(|arrival| {+ now.saturating_duration_since(*arrival) >= TILE_SETTLE_DURATION+ })+ {+ let phase = usize::try_from(phase_elapsed.as_millis() / FRAME_INTERVAL.as_millis())+ .unwrap_or(usize::MAX);+ let runway = (0..remaining.saturating_sub(/*rhs*/ 1).min(/*other*/ 3))+ .map(|index| {+ char::from(self.flap_sample[(phase + index) % self.flap_sample.len()])+ })+ .collect::<String>();+ line.line.spans.push(Span::styled(+ format!(" {runway}"),+ board_style.fg(Color::DarkGray),+ ));+ }+ let padding = usize::from(width).saturating_sub(line.width());+ if padding > 0 {+ line.line+ .spans+ .push(Span::styled(" ".repeat(padding), board_style));+ }+ }++ if self.is_animating(elapsed) {+ self.frame_requester.schedule_frame_in(FRAME_INTERVAL);+ }+ lines+ }+}++impl HistoryCell for SplitFlapTranscriptCell {+ fn display_lines(&self, width: u16) -> Vec<Line<'static>> {+ visible_lines(self.display_hyperlink_lines(width))+ }++ fn raw_lines(&self) -> Vec<Line<'static>> {+ self.inner.raw_lines()+ }++ fn display_hyperlink_lines(&self, width: u16) -> Vec<HyperlinkLine> {+ self.animate_lines(+ self.inner.display_hyperlink_lines(width),+ width,+ self.started_at.elapsed(),+ )+ }++ fn transcript_hyperlink_lines(&self, width: u16) -> Vec<HyperlinkLine> {+ self.animate_lines(+ self.inner.transcript_hyperlink_lines(width),+ width,+ self.started_at.elapsed(),+ )+ }++ fn transcript_animation_tick(&self) -> Option<u64> {+ let elapsed = self.started_at.elapsed();+ self.is_animating(elapsed).then(|| {+ u64::try_from(elapsed.as_millis() / FRAME_INTERVAL.as_millis()).unwrap_or(u64::MAX)+ })+ }+}++fn visible_glyphs(cell: &dyn HistoryCell) -> String {+ let mut glyphs = String::new();+ for line in cell.display_hyperlink_lines(u16::MAX) {+ for span in line.line.spans {+ for grapheme in span.content.graphemes(/*is_extended*/ true) {+ if is_flippable(grapheme) {+ glyphs.push_str(grapheme);+ }+ }+ }+ }+ glyphs+}++fn is_flippable(grapheme: &str) -> bool {+ grapheme.len() == 1 && grapheme.as_bytes()[0].is_ascii_alphanumeric()+}++fn is_punctuation(grapheme: &str) -> bool {+ grapheme.len() == 1 && grapheme.as_bytes()[0].is_ascii_punctuation()+}++fn flap_glyph(+ grapheme: &str,+ index: usize,+ elapsed: Duration,+ phase_elapsed: Duration,+ sample: &[u8],+) -> (String, bool) {+ let milliseconds = elapsed.as_millis();+ if milliseconds < 40 {+ return (" ".to_string(), true);+ }+ if elapsed >= TILE_SETTLE_DURATION {+ return (grapheme.to_string(), false);+ }++ let frame = usize::try_from(phase_elapsed.as_millis() / FRAME_INTERVAL.as_millis())+ .unwrap_or(usize::MAX);+ let target = usize::from(grapheme.as_bytes()[0]);+ let offset = target+ .wrapping_add(index.wrapping_mul(/*rhs*/ 7))+ .wrapping_add(frame.wrapping_mul(/*rhs*/ 11));+ let tile = char::from(sample[offset % sample.len()]);+ (tile.to_string(), true)+}diff --git a/codex-rs/tui/src/chatwidget/realtime_split_flap_tests.rs b/codex-rs/tui/src/chatwidget/realtime_split_flap_tests.rsindex 1214e12b54..b3c1597ced 100644--- a/codex-rs/tui/src/chatwidget/realtime_split_flap_tests.rs+++ b/codex-rs/tui/src/chatwidget/realtime_split_flap_tests.rs@@ -1,8 +1,217 @@//! Regressions for bounded state used by realtime voice presentation.//! Retained history must preserve ordering without growing beyond its limits.-use super::VoiceAmplitudeHistory;+use super::*;use pretty_assertions::assert_eq;+use ratatui::buffer::Buffer;+use ratatui::layout::Rect;+use ratatui::widgets::Paragraph;+use ratatui::widgets::Widget;++#[derive(Debug)]+struct PlainTranscriptCell(String);++impl HistoryCell for PlainTranscriptCell {+ fn display_lines(&self, _width: u16) -> Vec<Line<'static>> {+ vec![Line::from(vec![Span::raw("› "), Span::raw(self.0.clone())])]+ }++ fn raw_lines(&self) -> Vec<Line<'static>> {+ vec![Line::from(self.0.clone())]+ }+}++fn cell(text: &str, motion_mode: MotionMode) -> SplitFlapTranscriptCell {+ SplitFlapTranscriptCell::new(+ Box::new(PlainTranscriptCell(text.to_string())),+ "user",+ text,+ /*previous*/ None,+ motion_mode,+ FrameRequester::test_dummy(),+ )+}++fn frame(cell: &SplitFlapTranscriptCell, milliseconds: u64) -> String {+ cell.animate_lines(+ cell.inner.display_hyperlink_lines(/*width*/ 16),+ /*width*/ 16,+ Duration::from_millis(milliseconds),+ )+ .into_iter()+ .map(|line| line.line.to_string().trim_end().to_string())+ .collect::<Vec<_>>()+ .join("\n")+}++#[test]+fn split_flap_frames_assemble_a_transcript_from_dark_tiles() {+ let board = cell("GATE 73", MotionMode::Animated);+ let frames = [0, 45, 90, 180, 285, 675]+ .into_iter()+ .map(|milliseconds| format!("{milliseconds:>3}ms {}", frame(&board, milliseconds)))+ .collect::<Vec<_>>()+ .join("\n");++ insta::assert_snapshot!(frames, @r"+ 0ms ›+ 45ms › T+ 90ms › ATGG AG+ 180ms › GGTT ET GAT+ 285ms › GATE 73 TEG+ 675ms › GATE 73+ ");++ let punctuated = cell("HELLO, WORLD?!", MotionMode::Animated);+ assert!(!frame(&punctuated, /*milliseconds*/ 90).contains(','));+ assert!(!frame(&punctuated, /*milliseconds*/ 180).contains(','));+ assert_eq!(frame(&punctuated, /*milliseconds*/ 285), "› HELLO, WORLD?!");+ assert_ne!(+ frame(&board, /*milliseconds*/ 285),+ frame(&board, /*milliseconds*/ 330)+ );++ let lowercase = cell("please keep going", MotionMode::Animated);+ assert!(lowercase.flap_sample.iter().all(u8::is_ascii_lowercase));+ for elapsed in [90, 285] {+ assert!(+ frame(&lowercase, elapsed)+ .chars()+ .all(|ch| !ch.is_ascii_uppercase())+ );+ }+}++#[test]+fn split_flap_paints_the_entire_transcript_row_black() {+ let cell = cell("GATE", MotionMode::Animated);+ let lines = cell.animate_lines(+ cell.inner.display_hyperlink_lines(/*width*/ 12),+ /*width*/ 12,+ Duration::ZERO,+ );+ let area = Rect::new(+ /*x*/ 0, /*y*/ 0, /*width*/ 12, /*height*/ 1,+ );+ let mut buffer = Buffer::empty(area);+ Paragraph::new(visible_lines(lines)).render(area, &mut buffer);++ assert!(buffer.content.iter().all(|tile| tile.bg == Color::Black));+ assert_eq!(buffer[(0, 0)].fg, Color::Cyan);+}++#[test]+fn settled_tiles_briefly_glow_in_the_speakers_color() {+ for (role, motion_mode, expected) in [+ ("user", MotionMode::Animated, Some(Color::Cyan)),+ ("assistant", MotionMode::Animated, Some(Color::Magenta)),+ ("user", MotionMode::Reduced, None),+ ] {+ let cell = SplitFlapTranscriptCell::new(+ Box::new(PlainTranscriptCell("A".to_string())),+ role,+ "A",+ /*previous*/ None,+ motion_mode,+ FrameRequester::test_dummy(),+ );+ let tile_color = |elapsed| {+ cell.animate_lines(+ cell.inner.display_hyperlink_lines(/*width*/ 8),+ /*width*/ 8,+ elapsed,+ )[0]+ .line+ .spans+ .iter()+ .find(|span| span.content == "A")+ .and_then(|span| span.style.fg)+ };++ assert_eq!(tile_color(TILE_SETTLE_DURATION), expected);+ assert_eq!(+ tile_color(TILE_SETTLE_DURATION + TILE_AFTERGLOW_DURATION),+ (motion_mode == MotionMode::Animated).then_some(Color::Gray)+ );+ }+}++#[test]+fn reduced_motion_keeps_the_original_transcript_and_requests_no_frames() {+ let (frame_requester, mut requests) = FrameRequester::test_channel();+ let cell = SplitFlapTranscriptCell::new(+ Box::new(PlainTranscriptCell("GATE 73".to_string())),+ "user",+ "GATE 73",+ /*previous*/ None,+ MotionMode::Reduced,+ frame_requester,+ );++ assert_eq!(cell.display_lines(/*width*/ 16)[0].to_string(), "› GATE 73");+ assert_eq!(cell.transcript_animation_tick(), None);+ assert!(requests.try_recv().is_err());+}++#[test]+fn appended_transcript_keeps_the_previous_words_settled() {+ let mut first = cell("GATE", MotionMode::Animated);+ for arrival in &mut first.tile_arrivals {+ *arrival -= ANIMATION_DURATION;+ }+ let appended = SplitFlapTranscriptCell::new(+ Box::new(PlainTranscriptCell("GATE 73".to_string())),+ "user",+ "GATE 73",+ Some(&first),+ MotionMode::Animated,+ FrameRequester::test_dummy(),+ );++ assert!(frame(&appended, /*milliseconds*/ 0).starts_with("› GATE "));+ assert_eq!(appended.phase_started_at, first.phase_started_at);+ assert_eq!(appended.tile_arrivals[..4], first.tile_arrivals);+}++#[test]+fn unicode_graphemes_and_terminal_width_are_preserved() {+ let text = "a\u{301} 東京 👩💻 B";+ let cell = cell(text, MotionMode::Animated);+ let lines = cell.animate_lines(+ cell.inner.display_hyperlink_lines(/*width*/ 24),+ /*width*/ 24,+ Duration::ZERO,+ );++ assert!(lines[0].line.to_string().contains("a\u{301} 東京 👩💻"));+ assert_eq!(lines[0].width(), 24);+ assert_eq!(cell.raw_lines()[0].to_string(), text);+}++#[test]+fn settled_split_flap_stops_scheduling_animation_frames() {+ let (frame_requester, mut requests) = FrameRequester::test_channel();+ let mut cell = SplitFlapTranscriptCell::new(+ Box::new(PlainTranscriptCell("GATE".to_string())),+ "user",+ "GATE",+ /*previous*/ None,+ MotionMode::Animated,+ frame_requester,+ );+ assert!(requests.try_recv().is_ok());+ cell.started_at = Instant::now() - ANIMATION_DURATION;+ for arrival in &mut cell.tile_arrivals {+ *arrival -= ANIMATION_DURATION;+ }++ assert_eq!(cell.transcript_animation_tick(), None);+ assert_eq!(+ cell.display_lines(/*width*/ 12)[0].to_string().trim_end(),+ "› GATE"+ );+ assert!(requests.try_recv().is_err());+}#[test]fn voice_meter_shows_four_real_amplitude_samples_in_two_braille_cells() {diff --git a/codex-rs/tui/src/chatwidget/realtime_tests/caption_replay.rs b/codex-rs/tui/src/chatwidget/realtime_tests/caption_replay.rsindex 6aaa9d1130..c1956f016a 100644--- a/codex-rs/tui/src/chatwidget/realtime_tests/caption_replay.rs+++ b/codex-rs/tui/src/chatwidget/realtime_tests/caption_replay.rs@@ -327,6 +327,7 @@ async fn captioned_voice_answer_does_not_duplicate_on_close() {#[tokio::test]async fn restored_partial_caption_accepts_late_completion_without_duplicate_history() {let (mut chat, _sender, mut events, _ops) = make_chatwidget_manual_with_sender().await;+ chat.config.animations = false;chat.restore_realtime_transcript_cells(VecDeque::from([super::super::RealtimeTranscriptRecord {role: "user".into(),diff --git a/codex-rs/tui/src/chatwidget/realtime_tests/transcripts.rs b/codex-rs/tui/src/chatwidget/realtime_tests/transcripts.rsindex b49d31e4aa..39096aef9f 100644--- a/codex-rs/tui/src/chatwidget/realtime_tests/transcripts.rs+++ b/codex-rs/tui/src/chatwidget/realtime_tests/transcripts.rs@@ -526,6 +526,7 @@ async fn transcript_completion_waits_for_normal_agent_stream_consolidation() {#[tokio::test]async fn transcript_handoff_moves_deferred_repeats_and_partial_once() {let (mut chat, _sender, mut events, _ops) = make_chatwidget_manual_with_sender().await;+ chat.config.animations = false;activate_voice(&mut chat);chat.on_agent_message_delta("ordinary stream".to_string());for _ in 0..super::super::MAX_PENDING_TRANSCRIPT_CELLS {@@ -694,3 +695,34 @@ async fn completed_transcript_preserves_the_other_speakers_caption() {(Some("assistant"), "Still speaking"));}++#[tokio::test]+async fn live_voice_split_flap_animates_without_changing_final_history() {+ let (mut chat, _sender, mut events, _ops) = make_chatwidget_manual_with_sender().await;+ chat.config.animations = true;+ activate_voice(&mut chat);++ chat.on_realtime_transcript_delta("assistant".to_string(), "gate 73".to_string());++ let Some(cell) = chat.realtime_conversation.live_transcript_cell.as_ref() else {+ panic!("live voice transcripts should have an animation cell");+ };+ assert_eq!(cell.raw_lines()[0].to_string(), "gate 73");+ assert!(+ chat.active_cell_transcript_key()+ .is_some_and(|key| { key.animation_tick.is_some() })+ );++ chat.on_realtime_transcript_done("assistant".to_string(), "gate 73".to_string());++ assert!(chat.active_cell_transcript_key().is_none());+ let Ok(AppEvent::InsertHistoryCell(cell)) = events.try_recv() else {+ panic!("a completed voice transcript should commit ordinary history");+ };+ assert!(cell.transcript_animation_tick().is_none());+ assert!(+ cell.display_lines(/*width*/ 80)+ .iter()+ .any(|line| { line.to_string().contains("gate 73") })+ );+}