Lightweight coding agent that runs in your terminal
Rust
Omit undersized WAV output from Code Mode (#41934)
Jennifer Zhao981d6b652b71
diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lockindex 8a34615829..07b46199c9 100644--- a/codex-rs/Cargo.lock+++ b/codex-rs/Cargo.lock@@ -2638,6 +2638,7 @@ dependencies = [name = "codex-code-mode-runtime"version = "0.0.0"dependencies = [+ "base64 0.22.1","codex-code-mode-protocol","codex-protocol","deno_core_icudata",diff --git a/codex-rs/code-mode-runtime/Cargo.toml b/codex-rs/code-mode-runtime/Cargo.tomlindex e3708b657e..20c1e6cb4c 100644--- a/codex-rs/code-mode-runtime/Cargo.toml+++ b/codex-rs/code-mode-runtime/Cargo.toml@@ -13,6 +13,7 @@ path = "src/lib.rs"workspace = true[dependencies]+base64 = { workspace = true }codex-code-mode-protocol = { workspace = true }codex-protocol = { workspace = true }deno_core_icudata = { workspace = true }diff --git a/codex-rs/code-mode-runtime/src/runtime/audio.rs b/codex-rs/code-mode-runtime/src/runtime/audio.rsnew file mode 100644index 0000000000..c17b4bc4ba--- /dev/null+++ b/codex-rs/code-mode-runtime/src/runtime/audio.rs@@ -0,0 +1,63 @@+//! Measures tool-generated PCM WAV clips using the audio bytes actually present.+//! Unknown formats retain the existing audio output behavior.++use base64::Engine;+use base64::engine::general_purpose::STANDARD as BASE64_STANDARD;+use codex_protocol::models::MAX_PROMPT_AUDIO_INPUT_BYTES;++pub(super) fn wav_duration_seconds(audio_url: &str) -> Option<f64> {+ let (metadata, payload) = audio_url.split_once(',')?;+ if !metadata+ .split(';')+ .skip(1)+ .any(|part| part.eq_ignore_ascii_case("base64"))+ || payload.len() > MAX_PROMPT_AUDIO_INPUT_BYTES.div_ceil(3) * 4+ {+ return None;+ }+ let bytes = BASE64_STANDARD.decode(payload).ok()?;+ if bytes.get(..4)? != b"RIFF" || bytes.get(8..12)? != b"WAVE" {+ return None;+ }++ let mut chunks = bytes.get(12..)?;+ let mut format = None;+ while chunks.len() >= 8 {+ let chunk_id = &chunks[..4];+ let size = u32::from_le_bytes(chunks[4..8].try_into().ok()?) as usize;+ let remaining = &chunks[8..];+ // Streaming WAV headers can declare more data than the file contains.+ let chunk = &remaining[..size.min(remaining.len())];+ match chunk_id {+ b"fmt " => {+ let mut encoding = u16::from_le_bytes(chunk.get(..2)?.try_into().ok()?);+ if encoding == 0xfffe {+ // WAVE_FORMAT_EXTENSIBLE stores the encoding in a subtype GUID.+ if chunk.get(26..40)?+ != [0, 0, 0, 0, 0x10, 0, 0x80, 0, 0, 0xaa, 0, 0x38, 0x9b, 0x71]+ {+ return None;+ }+ encoding = u16::from_le_bytes(chunk.get(24..26)?.try_into().ok()?);+ }+ if !matches!(encoding, 1 | 3) {+ return None;+ }+ let sample_rate = u32::from_le_bytes(chunk.get(4..8)?.try_into().ok()?);+ let block_align = u16::from_le_bytes(chunk.get(12..14)?.try_into().ok()?);+ if sample_rate == 0 || block_align == 0 {+ return None;+ }+ format = Some((sample_rate, block_align));+ }+ b"data" => {+ let (sample_rate, block_align) = format?;+ let frames = chunk.len() / usize::from(block_align);+ return Some(frames as f64 / f64::from(sample_rate));+ }+ _ => {}+ }+ chunks = remaining.get(size.checked_add(size % 2)?..)?;+ }+ None+}diff --git a/codex-rs/code-mode-runtime/src/runtime/mod.rs b/codex-rs/code-mode-runtime/src/runtime/mod.rsindex 09e9f4aeed..3077b23330 100644--- a/codex-rs/code-mode-runtime/src/runtime/mod.rs+++ b/codex-rs/code-mode-runtime/src/runtime/mod.rs@@ -1,3 +1,4 @@+mod audio;mod callbacks;mod globals;mod module_loader;diff --git a/codex-rs/code-mode-runtime/src/runtime/value.rs b/codex-rs/code-mode-runtime/src/runtime/value.rsindex ac59b09ac3..f7e63d0429 100644--- a/codex-rs/code-mode-runtime/src/runtime/value.rs+++ b/codex-rs/code-mode-runtime/src/runtime/value.rs@@ -4,6 +4,8 @@ use codex_code_mode_protocol::DEFAULT_IMAGE_DETAIL;use codex_code_mode_protocol::FunctionCallOutputContentItem;use codex_code_mode_protocol::ImageDetail;+use super::audio::wav_duration_seconds;+const IMAGE_HELPER_EXPECTS_MESSAGE: &str = "image expects a non-empty image URL string, an object with image_url and optional detail, or a raw MCP image block";const AUDIO_HELPER_EXPECTS_MESSAGE: &str = "audio expects a non-empty audio URL string, an object with audio_url, or a raw MCP audio block";const REMOTE_IMAGE_URL_ERROR: &str = "Tool call failed: remote image URLs are not supported in tool outputs. Pass a base64 data URI instead";@@ -214,6 +216,14 @@ pub(super) fn normalize_output_audio(return Err(INVALID_AUDIO_URL_ERROR.to_string());}+ // Tiny tool-generated clips cannot be encoded reliably by audio models.+ if wav_duration_seconds(&audio_url).is_some_and(|duration| duration < 0.025) {+ return Ok(FunctionCallOutputContentItem::InputText {+ text: "Audio output omitted because the clip is shorter than 25 ms; use a longer clip."+ .to_string(),+ });+ }+Ok(FunctionCallOutputContentItem::InputAudio { audio_url })})();diff --git a/codex-rs/code-mode-runtime/src/service_audio_tests.rs b/codex-rs/code-mode-runtime/src/service_audio_tests.rsnew file mode 100644index 0000000000..45c1bc453a--- /dev/null+++ b/codex-rs/code-mode-runtime/src/service_audio_tests.rs@@ -0,0 +1,163 @@+//! Covers per-item short-WAV omissions without disrupting other Code Mode outputs.++use super::cell_id;+use super::execute;+use super::execute_request;+use crate::ExecuteRequest;+use crate::FunctionCallOutputContentItem;+use crate::InProcessCodeModeSession;+use crate::RuntimeResponse;+use crate::WaitOutcome;+use crate::WaitRequest;+use base64::Engine;+use base64::engine::general_purpose::STANDARD as BASE64_STANDARD;+use pretty_assertions::assert_eq;++const SHORT_AUDIO_OMISSION_TEXT: &str =+ "Audio output omitted because the clip is shorter than 25 ms; use a longer clip.";++fn pcm16_wav(sample_rate: u32, frames: usize) -> Vec<u8> {+ let data_size = u32::try_from(frames * 2).unwrap();+ let mut wav = b"RIFF".to_vec();+ wav.extend_from_slice(&(36 + data_size).to_le_bytes());+ wav.extend_from_slice(b"WAVEfmt ");+ wav.extend_from_slice(&16_u32.to_le_bytes());+ wav.extend_from_slice(&1_u16.to_le_bytes());+ wav.extend_from_slice(&1_u16.to_le_bytes());+ wav.extend_from_slice(&sample_rate.to_le_bytes());+ wav.extend_from_slice(&(sample_rate * 2).to_le_bytes());+ wav.extend_from_slice(&2_u16.to_le_bytes());+ wav.extend_from_slice(&16_u16.to_le_bytes());+ wav.extend_from_slice(b"data");+ wav.extend_from_slice(&data_size.to_le_bytes());+ wav.resize(44 + frames * 2, /*value*/ 0);+ wav+}++#[tokio::test]+async fn audio_helper_omits_short_wav_in_all_input_forms() {+ for (sample_rate, frames, omitted) in [+ (24_000, 0, true),+ (24_000, 1, true),+ (24_000, 599, true),+ (24_000, 600, false),+ (24_000, 601, false),+ (44_100, 223, true),+ (44_100, 1_102, true),+ (44_100, 1_103, false),+ (30_870, 223, true),+ (30_870, 771, true),+ (30_870, 772, false),+ (34_398, 221, true),+ (34_398, 859, true),+ (34_398, 860, false),+ ] {+ let payload = BASE64_STANDARD.encode(pcm16_wav(sample_rate, frames));+ let audio_url = format!("data:audio/wav;base64,{payload}");+ let service = InProcessCodeModeSession::new();+ let response = execute(+ &service,+ ExecuteRequest {+ source: format!(+ r#"audio({audio_url:?}); audio({{audio_url:{audio_url:?}}});+audio({{type:"audio",mimeType:"audio/wav",data:{payload:?}}});"#+ ),+ yield_time_ms: None,+ ..execute_request("")+ },+ )+ .await;+ let expected = if omitted {+ FunctionCallOutputContentItem::InputText {+ text: SHORT_AUDIO_OMISSION_TEXT.to_string(),+ }+ } else {+ FunctionCallOutputContentItem::InputAudio { audio_url }+ };+ assert_eq!(+ response,+ RuntimeResponse::Result {+ code_mode_host_duration: None,+ cell_id: cell_id("1"),+ content_items: vec![expected; 3],+ error_text: None,+ },+ "sample rate {sample_rate}, frames {frames}"+ );+ }+}++#[tokio::test]+async fn audio_helper_bounds_wav_data_and_preserves_outputs_after_yield() {+ let mut metadata_short = pcm16_wav(/*sample_rate*/ 24_000, /*frames*/ 1);+ let mut valid = pcm16_wav(/*sample_rate*/ 24_000, /*frames*/ 600);+ let mut list = b"LIST".to_vec();+ list.extend_from_slice(&(4_u32 + 8 + 2048).to_le_bytes());+ list.extend_from_slice(b"INFOINAM");+ list.extend_from_slice(&2048_u32.to_le_bytes());+ list.resize(8 + 4 + 8 + 2048, /*value*/ 0);+ list.extend_from_slice(b"JUNK\x01\0\0\0x\0");+ for wav in [&mut metadata_short, &mut valid] {+ drop(wav.splice(36..36, list.iter().copied()));+ let riff_size = u32::try_from(wav.len() - 8).unwrap();+ wav[4..8].copy_from_slice(&riff_size.to_le_bytes());+ }+ let mut truncated = pcm16_wav(/*sample_rate*/ 24_000, /*frames*/ 0);+ truncated[4..8].copy_from_slice(&48_036_u32.to_le_bytes());+ truncated[40..44].copy_from_slice(&48_000_u32.to_le_bytes());+ let mut bounded = pcm16_wav(/*sample_rate*/ 24_000, /*frames*/ 600);+ bounded[40..44].copy_from_slice(&2_u32.to_le_bytes());+ let urls = [metadata_short, truncated, bounded, valid]+ .map(|wav| format!("data:audio/wav;base64,{}", BASE64_STANDARD.encode(wav)));+ let audio_urls = serde_json::to_string(&urls).unwrap();+ let service = InProcessCodeModeSession::new();+ let response = execute(+ &service,+ ExecuteRequest {+ source: format!(+ "text('before'); await yield_control(); \+ for (const audio_url of {audio_urls}) audio({{audio_url}}); text('after');"+ ),+ yield_time_ms: None,+ ..execute_request("")+ },+ )+ .await;+ assert_eq!(+ response,+ RuntimeResponse::Yielded {+ code_mode_host_duration: None,+ cell_id: cell_id("1"),+ content_items: vec![FunctionCallOutputContentItem::InputText {+ text: "before".to_string(),+ }],+ }+ );+ let mut expected = vec![+ FunctionCallOutputContentItem::InputText {+ text: SHORT_AUDIO_OMISSION_TEXT.to_string(),+ };+ 3+ ];+ expected.push(FunctionCallOutputContentItem::InputAudio {+ audio_url: urls[3].clone(),+ });+ expected.push(FunctionCallOutputContentItem::InputText {+ text: "after".to_string(),+ });+ assert_eq!(+ service+ .wait(WaitRequest {+ cell_id: cell_id("1"),+ yield_time_ms: 60_000,+ })+ .await+ .unwrap(),+ WaitOutcome::LiveCell(RuntimeResponse::Result {+ code_mode_host_duration: None,+ cell_id: cell_id("1"),+ content_items: expected,+ error_text: None,+ })+ );+}diff --git a/codex-rs/code-mode-runtime/src/service_tests.rs b/codex-rs/code-mode-runtime/src/service_tests.rsindex c3c687b3ab..2ce272efe4 100644--- a/codex-rs/code-mode-runtime/src/service_tests.rs+++ b/codex-rs/code-mode-runtime/src/service_tests.rs@@ -1189,6 +1189,9 @@ audio({);}+#[path = "service_audio_tests.rs"]+mod audio_tests;+#[tokio::test]async fn audio_helper_rejects_non_data_urls() {for source in [diff --git a/codex-rs/core/tests/suite/code_mode.rs b/codex-rs/core/tests/suite/code_mode.rsindex 57f98df1d1..112740dfea 100644--- a/codex-rs/core/tests/suite/code_mode.rs+++ b/codex-rs/core/tests/suite/code_mode.rs@@ -38,6 +38,7 @@ use codex_protocol::models::ContentItem;use codex_protocol::models::ImageDetail;use codex_protocol::models::PermissionProfile;use codex_protocol::models::ResponseItem;+use codex_protocol::openai_models::InputModality;use codex_protocol::openai_models::ToolMode;use codex_protocol::protocol::AskForApproval;use codex_protocol::protocol::EventMsg;@@ -3856,6 +3857,89 @@ text("after");Ok(())}+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]+async fn code_mode_omits_short_audio_and_preserves_other_output() -> Result<()> {+ skip_if_no_network!(Ok(()));++ let [short_audio_url, valid_audio_url] = [120_u32, 600].map(|frames| {+ let sample_rate = 24_000_u32;+ let data_size = frames * 2;+ let mut wav = b"RIFF".to_vec();+ wav.extend_from_slice(&(36 + data_size).to_le_bytes());+ wav.extend_from_slice(b"WAVEfmt ");+ wav.extend_from_slice(&16_u32.to_le_bytes());+ wav.extend_from_slice(&1_u16.to_le_bytes());+ wav.extend_from_slice(&1_u16.to_le_bytes());+ wav.extend_from_slice(&sample_rate.to_le_bytes());+ wav.extend_from_slice(&(sample_rate * 2).to_le_bytes());+ wav.extend_from_slice(&2_u16.to_le_bytes());+ wav.extend_from_slice(&16_u16.to_le_bytes());+ wav.extend_from_slice(b"data");+ wav.extend_from_slice(&data_size.to_le_bytes());+ wav.resize(44 + data_size as usize, /*value*/ 0);+ format!("data:audio/wav;base64,{}", BASE64_STANDARD.encode(wav))+ });+ let code = format!(+ r#"+text("before");+audio({short_audio_url:?});+text("between");+audio({valid_audio_url:?});+text("after");+"#,+ );++ let server = responses::start_mock_server().await;+ let mut builder = test_codex()+ .with_model_info_override("gpt-5.4", |model| {+ model.input_modalities.push(InputModality::Audio);+ })+ .with_config(|config| {+ let _ = config.features.enable(Feature::CodeMode);+ });+ let test = builder.build_with_auto_env(&server).await?;+ responses::mount_sse_once(+ &server,+ sse(vec![+ ev_response_created("resp-1"),+ ev_custom_tool_call("call-1", "exec", &code),+ ev_completed("resp-1"),+ ]),+ )+ .await;+ let second_mock = responses::mount_sse_once(+ &server,+ sse(vec![+ ev_assistant_message("msg-1", "done"),+ ev_completed("resp-2"),+ ]),+ )+ .await;++ test.submit_turn("use exec to return short and valid audio with text")+ .await?;++ let req = second_mock.single_request();+ let items = custom_tool_output_items(&req, "call-1");+ let (_, success) = custom_tool_output_body_and_success(&req, "call-1");+ assert_ne!(success, Some(false));+ assert_eq!(+ &items[1..],+ &[+ serde_json::json!({ "type": "input_text", "text": "before" }),+ serde_json::json!({+ "type": "input_text",+ "text": "Audio output omitted because the clip is shorter than 25 ms; use a longer clip."+ }),+ serde_json::json!({ "type": "input_text", "text": "between" }),+ serde_json::json!({ "type": "input_audio", "audio_url": valid_audio_url }),+ serde_json::json!({ "type": "input_text", "text": "after" }),+ ]+ );++ Ok(())+}+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]async fn code_mode_replaces_malformed_image() -> Result<()> {skip_if_no_network!(Ok(()));