Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

Produce playable audio and choose provider-compatible voices after #1387

Bingxi Zhao (Frank)40b8e21487e2
diff --git a/deeptutor/capabilities/audio_overview/capability.py b/deeptutor/capabilities/audio_overview/capability.py
index 21adb0887..a9adad5ca 100644
--- a/deeptutor/capabilities/audio_overview/capability.py
+++ b/deeptutor/capabilities/audio_overview/capability.py
@@ -29,8 +29,8 @@ class AudioOverviewCapability(TurnCapability):
config_defaults={
"topic": "",
"target_minutes": 5,
- "host_voice": "alloy",
- "expert_voice": "nova",
+ "host_voice": None,
+ "expert_voice": None,
"max_context_chunks": 6,
},
)
diff --git a/deeptutor/capabilities/audio_overview/pipeline.py b/deeptutor/capabilities/audio_overview/pipeline.py
index 4abf94cfc..d5484512f 100644
--- a/deeptutor/capabilities/audio_overview/pipeline.py
+++ b/deeptutor/capabilities/audio_overview/pipeline.py
@@ -2,11 +2,16 @@
from __future__ import annotations
+import asyncio
from collections.abc import Awaitable, Callable, Mapping
from dataclasses import dataclass
+import io
from pathlib import Path
import re
+import subprocess
+import tempfile
from typing import Any
+import wave
from pydantic import BaseModel, ConfigDict, Field, ValidationError, model_validator
@@ -94,6 +99,103 @@ _LOCATOR_FIELDS = ("locator", "page", "page_number", "section", "url", "source_u
_SNIPPET_FIELDS = ("snippet", "content", "text", "preview", "excerpt")
+def _wave_parameters(audio: bytes) -> tuple[int, int, int] | None:
+ try:
+ with wave.open(io.BytesIO(audio), "rb") as source:
+ return source.getnchannels(), source.getsampwidth(), source.getframerate()
+ except (EOFError, wave.Error):
+ return None
+
+
+def _render_audio(parts: list[tuple[bytes, str]], output_dir: Path, stem: str) -> Path:
+ """Join decoded audio frames, never whole encoded files or mismatched MIME types."""
+ if not parts or any(not audio for audio, _ in parts):
+ raise AudioOverviewError("The speech provider returned empty audio.")
+ wave_params = [_wave_parameters(audio) for audio, _ in parts]
+ if all(params is not None and params == wave_params[0] for params in wave_params):
+ output = output_dir / f"{stem}.wav"
+ channels, sample_width, sample_rate = wave_params[0]
+ with wave.open(str(output), "wb") as target:
+ target.setnchannels(channels)
+ target.setsampwidth(sample_width)
+ target.setframerate(sample_rate)
+ for audio, _ in parts:
+ with wave.open(io.BytesIO(audio), "rb") as source:
+ target.writeframes(source.readframes(source.getnframes()))
+ return output
+
+ output = output_dir / f"{stem}.mp3"
+ with tempfile.TemporaryDirectory(prefix="audio-overview-") as directory:
+ raw_path = Path(directory) / "joined.pcm"
+ try:
+ with raw_path.open("wb") as raw:
+ for audio, content_type in parts:
+ input_args: list[str] = []
+ from deeptutor.services.voice.audio import _parse_pcm_content_type
+
+ pcm = _parse_pcm_content_type(content_type)
+ if pcm is not None:
+ sample_rate, channels = pcm
+ input_args = ["-f", "s16le", "-ar", str(sample_rate), "-ac", str(channels)]
+ decoded = subprocess.run(
+ [
+ "ffmpeg",
+ "-hide_banner",
+ "-loglevel",
+ "error",
+ "-nostdin",
+ *input_args,
+ "-i",
+ "pipe:0",
+ "-f",
+ "s16le",
+ "-ar",
+ "24000",
+ "-ac",
+ "1",
+ "pipe:1",
+ ],
+ input=audio,
+ capture_output=True,
+ check=False,
+ )
+ if decoded.returncode != 0 or not decoded.stdout:
+ raise AudioOverviewError(
+ "The speech provider returned audio that could not be decoded."
+ )
+ raw.write(decoded.stdout)
+ encoded = subprocess.run(
+ [
+ "ffmpeg",
+ "-hide_banner",
+ "-loglevel",
+ "error",
+ "-nostdin",
+ "-f",
+ "s16le",
+ "-ar",
+ "24000",
+ "-ac",
+ "1",
+ "-i",
+ str(raw_path),
+ "-codec:a",
+ "libmp3lame",
+ "-q:a",
+ "4",
+ "-y",
+ str(output),
+ ],
+ capture_output=True,
+ check=False,
+ )
+ except FileNotFoundError as exc:
+ raise AudioOverviewError("ffmpeg is required to join this speech format.") from exc
+ if encoded.returncode != 0 or not output.is_file() or output.stat().st_size == 0:
+ raise AudioOverviewError("Could not encode the audio overview.")
+ return output
+
+
def parse_script(raw_response: str, citation_ids: set[str]) -> AudioOverviewScript:
"""Parse and strictly validate the LLM script without repairing citations."""
@@ -311,28 +413,45 @@ class AudioOverviewPipeline:
speech = synthesize_speech_impl
- audio_parts: list[bytes] = []
- for segment in script.segments:
- voice = (
- request_config.host_voice
- if segment.speaker == "host"
- else request_config.expert_voice
+ host_voice = request_config.host_voice
+ expert_voice = request_config.expert_voice
+ if self._speech_func is None and (host_voice is None or expert_voice is None):
+ from deeptutor.services.config.provider_runtime import resolve_tts_runtime_config
+ from deeptutor.services.voice.options import voice_model_options
+
+ tts = resolve_tts_runtime_config()
+ choices = voice_model_options(tts.provider_name, "tts", tts.model).get("voices", [])
+ host_voice = host_voice or tts.voice
+ expert_voice = expert_voice or next(
+ (str(item["id"]) for item in choices if item.get("id") != host_voice), None
)
- audio, _content_type = await speech(
+ if not host_voice or not expert_voice:
+ raise AudioOverviewError(
+ "Configure distinct host and expert voices for the active TTS model."
+ )
+ else:
+ host_voice = host_voice or "host"
+ expert_voice = expert_voice or "expert"
+
+ audio_parts: list[tuple[bytes, str]] = []
+ for segment in script.segments:
+ voice = host_voice if segment.speaker == "host" else expert_voice
+ audio, content_type = await speech(
segment.text,
voice=voice,
response_format="mp3",
)
- audio_parts.append(audio)
+ audio_parts.append((audio, content_type))
stem = _safe_stem(f"{turn_id}-audio-overview", "audio-overview")
- audio_path = self.workspace_output_dir / f"{stem}.mp3"
transcript_path = self.workspace_output_dir / f"{stem}.md"
transcript = _transcript(script, context.citations, self.language)
try:
self.workspace_output_dir.mkdir(parents=True, exist_ok=True)
- audio_path.write_bytes(b"".join(audio_parts))
+ audio_path = await asyncio.to_thread(
+ _render_audio, audio_parts, self.workspace_output_dir, stem
+ )
transcript_path.write_text(transcript, encoding="utf-8")
except OSError as exc:
raise AudioOverviewError(f"Could not write audio overview artifacts: {exc}") from exc
diff --git a/deeptutor/capabilities/audio_overview/request_config.py b/deeptutor/capabilities/audio_overview/request_config.py
index f12a7b8c0..5cb78ca64 100644
--- a/deeptutor/capabilities/audio_overview/request_config.py
+++ b/deeptutor/capabilities/audio_overview/request_config.py
@@ -10,8 +10,8 @@ class AudioOverviewRequestConfig(BaseModel):
topic: str = Field(default="", max_length=500)
target_minutes: int = Field(default=5, ge=1, le=15)
- host_voice: str = Field(default="alloy", min_length=1, max_length=64)
- expert_voice: str = Field(default="nova", min_length=1, max_length=64)
+ host_voice: str | None = Field(default=None, min_length=1, max_length=64)
+ expert_voice: str | None = Field(default=None, min_length=1, max_length=64)
max_context_chunks: int = Field(default=6, ge=1, le=12)
diff --git a/tests/capabilities/test_audio_overview.py b/tests/capabilities/test_audio_overview.py
index 3a93f281d..dc328cd90 100644
--- a/tests/capabilities/test_audio_overview.py
+++ b/tests/capabilities/test_audio_overview.py
@@ -1,10 +1,14 @@
from __future__ import annotations
import asyncio
+import io
import json
from pathlib import Path
+import shutil
+import subprocess
from types import SimpleNamespace
from typing import Any
+import wave
import pytest
@@ -12,6 +16,7 @@ from deeptutor.capabilities.audio_overview.capability import AudioOverviewCapabi
from deeptutor.capabilities.audio_overview.pipeline import (
AudioOverviewError,
AudioOverviewPipeline,
+ _render_audio,
normalize_rag_result,
parse_script,
)
@@ -151,7 +156,13 @@ async def test_pipeline_generates_two_voices_and_publishes_workspace_items(
async def fake_speech(text: str, **kwargs: Any) -> tuple[bytes, str]:
speech_calls.append({"text": text, **kwargs})
- return (f"audio:{text}".encode(), "audio/mpeg")
+ buffer = io.BytesIO()
+ with wave.open(buffer, "wb") as output:
+ output.setnchannels(1)
+ output.setsampwidth(2)
+ output.setframerate(24000)
+ output.writeframes(b"\x00\x00" * 2400)
+ return buffer.getvalue(), "audio/wav"
published: list[list[dict[str, Any]]] = []
@@ -218,10 +229,12 @@ async def test_pipeline_generates_two_voices_and_publishes_workspace_items(
assert rag_calls == [("reliability audio overview", "engineering", 6)]
assert [call["voice"] for call in speech_calls] == ["alloy", "nova"]
- assert artifacts.audio_path.read_bytes() == b"audio:Let's begin.audio:Backoff controls retries."
+ with wave.open(str(artifacts.audio_path), "rb") as audio:
+ assert audio.getnframes() == 4800
+ assert audio.getframerate() == 24000
assert "[S1] Reliability.pdf" in artifacts.transcript_path.read_text(encoding="utf-8")
assert [row["path"] for row in published[0]] == [
- "outputs/audio_overview/session/turn/turn-42-audio-overview.mp3",
+ "outputs/audio_overview/session/turn/turn-42-audio-overview.wav",
"outputs/audio_overview/session/turn/turn-42-audio-overview.md",
]
assert [item["workspace_item_id"] for item in artifacts.workspace_items] == [
@@ -309,3 +322,40 @@ async def test_capability_publishes_two_workspace_attachments() -> None:
"transcript",
]
assert all(item["type"] == "workspace_item" for item in workspace_items)
+
+
+def test_mp3_segments_are_remuxed_into_one_playable_file(tmp_path: Path) -> None:
+ if shutil.which("ffmpeg") is None:
+ pytest.skip("ffmpeg is needed to verify MP3 remuxing")
+ buffer = io.BytesIO()
+ with wave.open(buffer, "wb") as output:
+ output.setnchannels(1)
+ output.setsampwidth(2)
+ output.setframerate(24000)
+ output.writeframes(b"\x00\x00" * 2400)
+ mp3 = subprocess.run(
+ [
+ "ffmpeg",
+ "-hide_banner",
+ "-loglevel",
+ "error",
+ "-f",
+ "wav",
+ "-i",
+ "pipe:0",
+ "-f",
+ "mp3",
+ "pipe:1",
+ ],
+ input=buffer.getvalue(),
+ capture_output=True,
+ check=True,
+ ).stdout
+ path = _render_audio([(mp3, "audio/mpeg"), (mp3, "audio/mpeg")], tmp_path, "overview")
+ assert path.suffix == ".mp3"
+ decoded = subprocess.run(
+ ["ffmpeg", "-hide_banner", "-loglevel", "error", "-i", str(path), "-f", "null", "-"],
+ capture_output=True,
+ check=False,
+ )
+ assert decoded.returncode == 0