Skip to content

armgit / codex

Public Imported from GitHub
0

Lightweight coding agent that runs in your terminal

Rust

Omit undersized WAV output from Code Mode (#41934)

Jennifer Zhao981d6b652b71
diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock
index 8a34615829..07b46199c9 100644
--- a/codex-rs/Cargo.lock
+++ b/codex-rs/Cargo.lock
@@ -2638,6 +2638,7 @@ dependencies = [
name = "codex-code-mode-runtime"
version = "0.0.0"
dependencies = [
+ "base64 0.22.1",
"codex-code-mode-protocol",
"codex-protocol",
"deno_core_icudata",
diff --git a/codex-rs/code-mode-runtime/Cargo.toml b/codex-rs/code-mode-runtime/Cargo.toml
index e3708b657e..20c1e6cb4c 100644
--- a/codex-rs/code-mode-runtime/Cargo.toml
+++ b/codex-rs/code-mode-runtime/Cargo.toml
@@ -13,6 +13,7 @@ path = "src/lib.rs"
workspace = true
[dependencies]
+base64 = { workspace = true }
codex-code-mode-protocol = { workspace = true }
codex-protocol = { workspace = true }
deno_core_icudata = { workspace = true }
diff --git a/codex-rs/code-mode-runtime/src/runtime/audio.rs b/codex-rs/code-mode-runtime/src/runtime/audio.rs
new file mode 100644
index 0000000000..c17b4bc4ba
--- /dev/null
+++ b/codex-rs/code-mode-runtime/src/runtime/audio.rs
@@ -0,0 +1,63 @@
+//! Measures tool-generated PCM WAV clips using the audio bytes actually present.
+//! Unknown formats retain the existing audio output behavior.
+
+use base64::Engine;
+use base64::engine::general_purpose::STANDARD as BASE64_STANDARD;
+use codex_protocol::models::MAX_PROMPT_AUDIO_INPUT_BYTES;
+
+pub(super) fn wav_duration_seconds(audio_url: &str) -> Option<f64> {
+ let (metadata, payload) = audio_url.split_once(',')?;
+ if !metadata
+ .split(';')
+ .skip(1)
+ .any(|part| part.eq_ignore_ascii_case("base64"))
+ || payload.len() > MAX_PROMPT_AUDIO_INPUT_BYTES.div_ceil(3) * 4
+ {
+ return None;
+ }
+ let bytes = BASE64_STANDARD.decode(payload).ok()?;
+ if bytes.get(..4)? != b"RIFF" || bytes.get(8..12)? != b"WAVE" {
+ return None;
+ }
+
+ let mut chunks = bytes.get(12..)?;
+ let mut format = None;
+ while chunks.len() >= 8 {
+ let chunk_id = &chunks[..4];
+ let size = u32::from_le_bytes(chunks[4..8].try_into().ok()?) as usize;
+ let remaining = &chunks[8..];
+ // Streaming WAV headers can declare more data than the file contains.
+ let chunk = &remaining[..size.min(remaining.len())];
+ match chunk_id {
+ b"fmt " => {
+ let mut encoding = u16::from_le_bytes(chunk.get(..2)?.try_into().ok()?);
+ if encoding == 0xfffe {
+ // WAVE_FORMAT_EXTENSIBLE stores the encoding in a subtype GUID.
+ if chunk.get(26..40)?
+ != [0, 0, 0, 0, 0x10, 0, 0x80, 0, 0, 0xaa, 0, 0x38, 0x9b, 0x71]
+ {
+ return None;
+ }
+ encoding = u16::from_le_bytes(chunk.get(24..26)?.try_into().ok()?);
+ }
+ if !matches!(encoding, 1 | 3) {
+ return None;
+ }
+ let sample_rate = u32::from_le_bytes(chunk.get(4..8)?.try_into().ok()?);
+ let block_align = u16::from_le_bytes(chunk.get(12..14)?.try_into().ok()?);
+ if sample_rate == 0 || block_align == 0 {
+ return None;
+ }
+ format = Some((sample_rate, block_align));
+ }
+ b"data" => {
+ let (sample_rate, block_align) = format?;
+ let frames = chunk.len() / usize::from(block_align);
+ return Some(frames as f64 / f64::from(sample_rate));
+ }
+ _ => {}
+ }
+ chunks = remaining.get(size.checked_add(size % 2)?..)?;
+ }
+ None
+}
diff --git a/codex-rs/code-mode-runtime/src/runtime/mod.rs b/codex-rs/code-mode-runtime/src/runtime/mod.rs
index 09e9f4aeed..3077b23330 100644
--- a/codex-rs/code-mode-runtime/src/runtime/mod.rs
+++ b/codex-rs/code-mode-runtime/src/runtime/mod.rs
@@ -1,3 +1,4 @@
+mod audio;
mod callbacks;
mod globals;
mod module_loader;
diff --git a/codex-rs/code-mode-runtime/src/runtime/value.rs b/codex-rs/code-mode-runtime/src/runtime/value.rs
index ac59b09ac3..f7e63d0429 100644
--- a/codex-rs/code-mode-runtime/src/runtime/value.rs
+++ b/codex-rs/code-mode-runtime/src/runtime/value.rs
@@ -4,6 +4,8 @@ use codex_code_mode_protocol::DEFAULT_IMAGE_DETAIL;
use codex_code_mode_protocol::FunctionCallOutputContentItem;
use codex_code_mode_protocol::ImageDetail;
+use super::audio::wav_duration_seconds;
+
const IMAGE_HELPER_EXPECTS_MESSAGE: &str = "image expects a non-empty image URL string, an object with image_url and optional detail, or a raw MCP image block";
const AUDIO_HELPER_EXPECTS_MESSAGE: &str = "audio expects a non-empty audio URL string, an object with audio_url, or a raw MCP audio block";
const REMOTE_IMAGE_URL_ERROR: &str = "Tool call failed: remote image URLs are not supported in tool outputs. Pass a base64 data URI instead";
@@ -214,6 +216,14 @@ pub(super) fn normalize_output_audio(
return Err(INVALID_AUDIO_URL_ERROR.to_string());
}
+ // Tiny tool-generated clips cannot be encoded reliably by audio models.
+ if wav_duration_seconds(&audio_url).is_some_and(|duration| duration < 0.025) {
+ return Ok(FunctionCallOutputContentItem::InputText {
+ text: "Audio output omitted because the clip is shorter than 25 ms; use a longer clip."
+ .to_string(),
+ });
+ }
+
Ok(FunctionCallOutputContentItem::InputAudio { audio_url })
})();
diff --git a/codex-rs/code-mode-runtime/src/service_audio_tests.rs b/codex-rs/code-mode-runtime/src/service_audio_tests.rs
new file mode 100644
index 0000000000..45c1bc453a
--- /dev/null
+++ b/codex-rs/code-mode-runtime/src/service_audio_tests.rs
@@ -0,0 +1,163 @@
+//! Covers per-item short-WAV omissions without disrupting other Code Mode outputs.
+
+use super::cell_id;
+use super::execute;
+use super::execute_request;
+use crate::ExecuteRequest;
+use crate::FunctionCallOutputContentItem;
+use crate::InProcessCodeModeSession;
+use crate::RuntimeResponse;
+use crate::WaitOutcome;
+use crate::WaitRequest;
+use base64::Engine;
+use base64::engine::general_purpose::STANDARD as BASE64_STANDARD;
+use pretty_assertions::assert_eq;
+
+const SHORT_AUDIO_OMISSION_TEXT: &str =
+ "Audio output omitted because the clip is shorter than 25 ms; use a longer clip.";
+
+fn pcm16_wav(sample_rate: u32, frames: usize) -> Vec<u8> {
+ let data_size = u32::try_from(frames * 2).unwrap();
+ let mut wav = b"RIFF".to_vec();
+ wav.extend_from_slice(&(36 + data_size).to_le_bytes());
+ wav.extend_from_slice(b"WAVEfmt ");
+ wav.extend_from_slice(&16_u32.to_le_bytes());
+ wav.extend_from_slice(&1_u16.to_le_bytes());
+ wav.extend_from_slice(&1_u16.to_le_bytes());
+ wav.extend_from_slice(&sample_rate.to_le_bytes());
+ wav.extend_from_slice(&(sample_rate * 2).to_le_bytes());
+ wav.extend_from_slice(&2_u16.to_le_bytes());
+ wav.extend_from_slice(&16_u16.to_le_bytes());
+ wav.extend_from_slice(b"data");
+ wav.extend_from_slice(&data_size.to_le_bytes());
+ wav.resize(44 + frames * 2, /*value*/ 0);
+ wav
+}
+
+#[tokio::test]
+async fn audio_helper_omits_short_wav_in_all_input_forms() {
+ for (sample_rate, frames, omitted) in [
+ (24_000, 0, true),
+ (24_000, 1, true),
+ (24_000, 599, true),
+ (24_000, 600, false),
+ (24_000, 601, false),
+ (44_100, 223, true),
+ (44_100, 1_102, true),
+ (44_100, 1_103, false),
+ (30_870, 223, true),
+ (30_870, 771, true),
+ (30_870, 772, false),
+ (34_398, 221, true),
+ (34_398, 859, true),
+ (34_398, 860, false),
+ ] {
+ let payload = BASE64_STANDARD.encode(pcm16_wav(sample_rate, frames));
+ let audio_url = format!("data:audio/wav;base64,{payload}");
+ let service = InProcessCodeModeSession::new();
+ let response = execute(
+ &service,
+ ExecuteRequest {
+ source: format!(
+ r#"audio({audio_url:?}); audio({{audio_url:{audio_url:?}}});
+audio({{type:"audio",mimeType:"audio/wav",data:{payload:?}}});"#
+ ),
+ yield_time_ms: None,
+ ..execute_request("")
+ },
+ )
+ .await;
+ let expected = if omitted {
+ FunctionCallOutputContentItem::InputText {
+ text: SHORT_AUDIO_OMISSION_TEXT.to_string(),
+ }
+ } else {
+ FunctionCallOutputContentItem::InputAudio { audio_url }
+ };
+ assert_eq!(
+ response,
+ RuntimeResponse::Result {
+ code_mode_host_duration: None,
+ cell_id: cell_id("1"),
+ content_items: vec![expected; 3],
+ error_text: None,
+ },
+ "sample rate {sample_rate}, frames {frames}"
+ );
+ }
+}
+
+#[tokio::test]
+async fn audio_helper_bounds_wav_data_and_preserves_outputs_after_yield() {
+ let mut metadata_short = pcm16_wav(/*sample_rate*/ 24_000, /*frames*/ 1);
+ let mut valid = pcm16_wav(/*sample_rate*/ 24_000, /*frames*/ 600);
+ let mut list = b"LIST".to_vec();
+ list.extend_from_slice(&(4_u32 + 8 + 2048).to_le_bytes());
+ list.extend_from_slice(b"INFOINAM");
+ list.extend_from_slice(&2048_u32.to_le_bytes());
+ list.resize(8 + 4 + 8 + 2048, /*value*/ 0);
+ list.extend_from_slice(b"JUNK\x01\0\0\0x\0");
+ for wav in [&mut metadata_short, &mut valid] {
+ drop(wav.splice(36..36, list.iter().copied()));
+ let riff_size = u32::try_from(wav.len() - 8).unwrap();
+ wav[4..8].copy_from_slice(&riff_size.to_le_bytes());
+ }
+ let mut truncated = pcm16_wav(/*sample_rate*/ 24_000, /*frames*/ 0);
+ truncated[4..8].copy_from_slice(&48_036_u32.to_le_bytes());
+ truncated[40..44].copy_from_slice(&48_000_u32.to_le_bytes());
+ let mut bounded = pcm16_wav(/*sample_rate*/ 24_000, /*frames*/ 600);
+ bounded[40..44].copy_from_slice(&2_u32.to_le_bytes());
+ let urls = [metadata_short, truncated, bounded, valid]
+ .map(|wav| format!("data:audio/wav;base64,{}", BASE64_STANDARD.encode(wav)));
+ let audio_urls = serde_json::to_string(&urls).unwrap();
+ let service = InProcessCodeModeSession::new();
+ let response = execute(
+ &service,
+ ExecuteRequest {
+ source: format!(
+ "text('before'); await yield_control(); \
+ for (const audio_url of {audio_urls}) audio({{audio_url}}); text('after');"
+ ),
+ yield_time_ms: None,
+ ..execute_request("")
+ },
+ )
+ .await;
+ assert_eq!(
+ response,
+ RuntimeResponse::Yielded {
+ code_mode_host_duration: None,
+ cell_id: cell_id("1"),
+ content_items: vec![FunctionCallOutputContentItem::InputText {
+ text: "before".to_string(),
+ }],
+ }
+ );
+ let mut expected = vec![
+ FunctionCallOutputContentItem::InputText {
+ text: SHORT_AUDIO_OMISSION_TEXT.to_string(),
+ };
+ 3
+ ];
+ expected.push(FunctionCallOutputContentItem::InputAudio {
+ audio_url: urls[3].clone(),
+ });
+ expected.push(FunctionCallOutputContentItem::InputText {
+ text: "after".to_string(),
+ });
+ assert_eq!(
+ service
+ .wait(WaitRequest {
+ cell_id: cell_id("1"),
+ yield_time_ms: 60_000,
+ })
+ .await
+ .unwrap(),
+ WaitOutcome::LiveCell(RuntimeResponse::Result {
+ code_mode_host_duration: None,
+ cell_id: cell_id("1"),
+ content_items: expected,
+ error_text: None,
+ })
+ );
+}
diff --git a/codex-rs/code-mode-runtime/src/service_tests.rs b/codex-rs/code-mode-runtime/src/service_tests.rs
index c3c687b3ab..2ce272efe4 100644
--- a/codex-rs/code-mode-runtime/src/service_tests.rs
+++ b/codex-rs/code-mode-runtime/src/service_tests.rs
@@ -1189,6 +1189,9 @@ audio({
);
}
+#[path = "service_audio_tests.rs"]
+mod audio_tests;
+
#[tokio::test]
async fn audio_helper_rejects_non_data_urls() {
for source in [
diff --git a/codex-rs/core/tests/suite/code_mode.rs b/codex-rs/core/tests/suite/code_mode.rs
index 57f98df1d1..112740dfea 100644
--- a/codex-rs/core/tests/suite/code_mode.rs
+++ b/codex-rs/core/tests/suite/code_mode.rs
@@ -38,6 +38,7 @@ use codex_protocol::models::ContentItem;
use codex_protocol::models::ImageDetail;
use codex_protocol::models::PermissionProfile;
use codex_protocol::models::ResponseItem;
+use codex_protocol::openai_models::InputModality;
use codex_protocol::openai_models::ToolMode;
use codex_protocol::protocol::AskForApproval;
use codex_protocol::protocol::EventMsg;
@@ -3856,6 +3857,89 @@ text("after");
Ok(())
}
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn code_mode_omits_short_audio_and_preserves_other_output() -> Result<()> {
+ skip_if_no_network!(Ok(()));
+
+ let [short_audio_url, valid_audio_url] = [120_u32, 600].map(|frames| {
+ let sample_rate = 24_000_u32;
+ let data_size = frames * 2;
+ let mut wav = b"RIFF".to_vec();
+ wav.extend_from_slice(&(36 + data_size).to_le_bytes());
+ wav.extend_from_slice(b"WAVEfmt ");
+ wav.extend_from_slice(&16_u32.to_le_bytes());
+ wav.extend_from_slice(&1_u16.to_le_bytes());
+ wav.extend_from_slice(&1_u16.to_le_bytes());
+ wav.extend_from_slice(&sample_rate.to_le_bytes());
+ wav.extend_from_slice(&(sample_rate * 2).to_le_bytes());
+ wav.extend_from_slice(&2_u16.to_le_bytes());
+ wav.extend_from_slice(&16_u16.to_le_bytes());
+ wav.extend_from_slice(b"data");
+ wav.extend_from_slice(&data_size.to_le_bytes());
+ wav.resize(44 + data_size as usize, /*value*/ 0);
+ format!("data:audio/wav;base64,{}", BASE64_STANDARD.encode(wav))
+ });
+ let code = format!(
+ r#"
+text("before");
+audio({short_audio_url:?});
+text("between");
+audio({valid_audio_url:?});
+text("after");
+"#,
+ );
+
+ let server = responses::start_mock_server().await;
+ let mut builder = test_codex()
+ .with_model_info_override("gpt-5.4", |model| {
+ model.input_modalities.push(InputModality::Audio);
+ })
+ .with_config(|config| {
+ let _ = config.features.enable(Feature::CodeMode);
+ });
+ let test = builder.build_with_auto_env(&server).await?;
+ responses::mount_sse_once(
+ &server,
+ sse(vec![
+ ev_response_created("resp-1"),
+ ev_custom_tool_call("call-1", "exec", &code),
+ ev_completed("resp-1"),
+ ]),
+ )
+ .await;
+ let second_mock = responses::mount_sse_once(
+ &server,
+ sse(vec![
+ ev_assistant_message("msg-1", "done"),
+ ev_completed("resp-2"),
+ ]),
+ )
+ .await;
+
+ test.submit_turn("use exec to return short and valid audio with text")
+ .await?;
+
+ let req = second_mock.single_request();
+ let items = custom_tool_output_items(&req, "call-1");
+ let (_, success) = custom_tool_output_body_and_success(&req, "call-1");
+ assert_ne!(success, Some(false));
+ assert_eq!(
+ &items[1..],
+ &[
+ serde_json::json!({ "type": "input_text", "text": "before" }),
+ serde_json::json!({
+ "type": "input_text",
+ "text": "Audio output omitted because the clip is shorter than 25 ms; use a longer clip."
+ }),
+ serde_json::json!({ "type": "input_text", "text": "between" }),
+ serde_json::json!({ "type": "input_audio", "audio_url": valid_audio_url }),
+ serde_json::json!({ "type": "input_text", "text": "after" }),
+ ]
+ );
+
+ Ok(())
+}
+
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn code_mode_replaces_malformed_image() -> Result<()> {
skip_if_no_network!(Ok(()));