DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.
Python ai-agentsai-tutorclawbotcli-tooldeepresearchinteractive-learninglearningmulti-agent-systemsrag
Produce playable audio and choose provider-compatible voices after #1387
Bingxi Zhao (Frank)40b8e21487e2
diff --git a/deeptutor/capabilities/audio_overview/capability.py b/deeptutor/capabilities/audio_overview/capability.pyindex 21adb0887..a9adad5ca 100644--- a/deeptutor/capabilities/audio_overview/capability.py+++ b/deeptutor/capabilities/audio_overview/capability.py@@ -29,8 +29,8 @@ class AudioOverviewCapability(TurnCapability):config_defaults={"topic": "","target_minutes": 5,- "host_voice": "alloy",- "expert_voice": "nova",+ "host_voice": None,+ "expert_voice": None,"max_context_chunks": 6,},)diff --git a/deeptutor/capabilities/audio_overview/pipeline.py b/deeptutor/capabilities/audio_overview/pipeline.pyindex 4abf94cfc..d5484512f 100644--- a/deeptutor/capabilities/audio_overview/pipeline.py+++ b/deeptutor/capabilities/audio_overview/pipeline.py@@ -2,11 +2,16 @@from __future__ import annotations+import asynciofrom collections.abc import Awaitable, Callable, Mappingfrom dataclasses import dataclass+import iofrom pathlib import Pathimport re+import subprocess+import tempfilefrom typing import Any+import wavefrom pydantic import BaseModel, ConfigDict, Field, ValidationError, model_validator@@ -94,6 +99,103 @@ _LOCATOR_FIELDS = ("locator", "page", "page_number", "section", "url", "source_u_SNIPPET_FIELDS = ("snippet", "content", "text", "preview", "excerpt")+def _wave_parameters(audio: bytes) -> tuple[int, int, int] | None:+ try:+ with wave.open(io.BytesIO(audio), "rb") as source:+ return source.getnchannels(), source.getsampwidth(), source.getframerate()+ except (EOFError, wave.Error):+ return None+++def _render_audio(parts: list[tuple[bytes, str]], output_dir: Path, stem: str) -> Path:+ """Join decoded audio frames, never whole encoded files or mismatched MIME types."""+ if not parts or any(not audio for audio, _ in parts):+ raise AudioOverviewError("The speech provider returned empty audio.")+ wave_params = [_wave_parameters(audio) for audio, _ in parts]+ if all(params is not None and params == wave_params[0] for params in wave_params):+ output = output_dir / f"{stem}.wav"+ channels, sample_width, sample_rate = wave_params[0]+ with wave.open(str(output), "wb") as target:+ target.setnchannels(channels)+ target.setsampwidth(sample_width)+ target.setframerate(sample_rate)+ for audio, _ in parts:+ with wave.open(io.BytesIO(audio), "rb") as source:+ target.writeframes(source.readframes(source.getnframes()))+ return output++ output = output_dir / f"{stem}.mp3"+ with tempfile.TemporaryDirectory(prefix="audio-overview-") as directory:+ raw_path = Path(directory) / "joined.pcm"+ try:+ with raw_path.open("wb") as raw:+ for audio, content_type in parts:+ input_args: list[str] = []+ from deeptutor.services.voice.audio import _parse_pcm_content_type++ pcm = _parse_pcm_content_type(content_type)+ if pcm is not None:+ sample_rate, channels = pcm+ input_args = ["-f", "s16le", "-ar", str(sample_rate), "-ac", str(channels)]+ decoded = subprocess.run(+ [+ "ffmpeg",+ "-hide_banner",+ "-loglevel",+ "error",+ "-nostdin",+ *input_args,+ "-i",+ "pipe:0",+ "-f",+ "s16le",+ "-ar",+ "24000",+ "-ac",+ "1",+ "pipe:1",+ ],+ input=audio,+ capture_output=True,+ check=False,+ )+ if decoded.returncode != 0 or not decoded.stdout:+ raise AudioOverviewError(+ "The speech provider returned audio that could not be decoded."+ )+ raw.write(decoded.stdout)+ encoded = subprocess.run(+ [+ "ffmpeg",+ "-hide_banner",+ "-loglevel",+ "error",+ "-nostdin",+ "-f",+ "s16le",+ "-ar",+ "24000",+ "-ac",+ "1",+ "-i",+ str(raw_path),+ "-codec:a",+ "libmp3lame",+ "-q:a",+ "4",+ "-y",+ str(output),+ ],+ capture_output=True,+ check=False,+ )+ except FileNotFoundError as exc:+ raise AudioOverviewError("ffmpeg is required to join this speech format.") from exc+ if encoded.returncode != 0 or not output.is_file() or output.stat().st_size == 0:+ raise AudioOverviewError("Could not encode the audio overview.")+ return output++def parse_script(raw_response: str, citation_ids: set[str]) -> AudioOverviewScript:"""Parse and strictly validate the LLM script without repairing citations."""@@ -311,28 +413,45 @@ class AudioOverviewPipeline:speech = synthesize_speech_impl- audio_parts: list[bytes] = []- for segment in script.segments:- voice = (- request_config.host_voice- if segment.speaker == "host"- else request_config.expert_voice+ host_voice = request_config.host_voice+ expert_voice = request_config.expert_voice+ if self._speech_func is None and (host_voice is None or expert_voice is None):+ from deeptutor.services.config.provider_runtime import resolve_tts_runtime_config+ from deeptutor.services.voice.options import voice_model_options++ tts = resolve_tts_runtime_config()+ choices = voice_model_options(tts.provider_name, "tts", tts.model).get("voices", [])+ host_voice = host_voice or tts.voice+ expert_voice = expert_voice or next(+ (str(item["id"]) for item in choices if item.get("id") != host_voice), None)- audio, _content_type = await speech(+ if not host_voice or not expert_voice:+ raise AudioOverviewError(+ "Configure distinct host and expert voices for the active TTS model."+ )+ else:+ host_voice = host_voice or "host"+ expert_voice = expert_voice or "expert"++ audio_parts: list[tuple[bytes, str]] = []+ for segment in script.segments:+ voice = host_voice if segment.speaker == "host" else expert_voice+ audio, content_type = await speech(segment.text,voice=voice,response_format="mp3",)- audio_parts.append(audio)+ audio_parts.append((audio, content_type))stem = _safe_stem(f"{turn_id}-audio-overview", "audio-overview")- audio_path = self.workspace_output_dir / f"{stem}.mp3"transcript_path = self.workspace_output_dir / f"{stem}.md"transcript = _transcript(script, context.citations, self.language)try:self.workspace_output_dir.mkdir(parents=True, exist_ok=True)- audio_path.write_bytes(b"".join(audio_parts))+ audio_path = await asyncio.to_thread(+ _render_audio, audio_parts, self.workspace_output_dir, stem+ )transcript_path.write_text(transcript, encoding="utf-8")except OSError as exc:raise AudioOverviewError(f"Could not write audio overview artifacts: {exc}") from excdiff --git a/deeptutor/capabilities/audio_overview/request_config.py b/deeptutor/capabilities/audio_overview/request_config.pyindex f12a7b8c0..5cb78ca64 100644--- a/deeptutor/capabilities/audio_overview/request_config.py+++ b/deeptutor/capabilities/audio_overview/request_config.py@@ -10,8 +10,8 @@ class AudioOverviewRequestConfig(BaseModel):topic: str = Field(default="", max_length=500)target_minutes: int = Field(default=5, ge=1, le=15)- host_voice: str = Field(default="alloy", min_length=1, max_length=64)- expert_voice: str = Field(default="nova", min_length=1, max_length=64)+ host_voice: str | None = Field(default=None, min_length=1, max_length=64)+ expert_voice: str | None = Field(default=None, min_length=1, max_length=64)max_context_chunks: int = Field(default=6, ge=1, le=12)diff --git a/tests/capabilities/test_audio_overview.py b/tests/capabilities/test_audio_overview.pyindex 3a93f281d..dc328cd90 100644--- a/tests/capabilities/test_audio_overview.py+++ b/tests/capabilities/test_audio_overview.py@@ -1,10 +1,14 @@from __future__ import annotationsimport asyncio+import ioimport jsonfrom pathlib import Path+import shutil+import subprocessfrom types import SimpleNamespacefrom typing import Any+import waveimport pytest@@ -12,6 +16,7 @@ from deeptutor.capabilities.audio_overview.capability import AudioOverviewCapabifrom deeptutor.capabilities.audio_overview.pipeline import (AudioOverviewError,AudioOverviewPipeline,+ _render_audio,normalize_rag_result,parse_script,)@@ -151,7 +156,13 @@ async def test_pipeline_generates_two_voices_and_publishes_workspace_items(async def fake_speech(text: str, **kwargs: Any) -> tuple[bytes, str]:speech_calls.append({"text": text, **kwargs})- return (f"audio:{text}".encode(), "audio/mpeg")+ buffer = io.BytesIO()+ with wave.open(buffer, "wb") as output:+ output.setnchannels(1)+ output.setsampwidth(2)+ output.setframerate(24000)+ output.writeframes(b"\x00\x00" * 2400)+ return buffer.getvalue(), "audio/wav"published: list[list[dict[str, Any]]] = []@@ -218,10 +229,12 @@ async def test_pipeline_generates_two_voices_and_publishes_workspace_items(assert rag_calls == [("reliability audio overview", "engineering", 6)]assert [call["voice"] for call in speech_calls] == ["alloy", "nova"]- assert artifacts.audio_path.read_bytes() == b"audio:Let's begin.audio:Backoff controls retries."+ with wave.open(str(artifacts.audio_path), "rb") as audio:+ assert audio.getnframes() == 4800+ assert audio.getframerate() == 24000assert "[S1] Reliability.pdf" in artifacts.transcript_path.read_text(encoding="utf-8")assert [row["path"] for row in published[0]] == [- "outputs/audio_overview/session/turn/turn-42-audio-overview.mp3",+ "outputs/audio_overview/session/turn/turn-42-audio-overview.wav","outputs/audio_overview/session/turn/turn-42-audio-overview.md",]assert [item["workspace_item_id"] for item in artifacts.workspace_items] == [@@ -309,3 +322,40 @@ async def test_capability_publishes_two_workspace_attachments() -> None:"transcript",]assert all(item["type"] == "workspace_item" for item in workspace_items)+++def test_mp3_segments_are_remuxed_into_one_playable_file(tmp_path: Path) -> None:+ if shutil.which("ffmpeg") is None:+ pytest.skip("ffmpeg is needed to verify MP3 remuxing")+ buffer = io.BytesIO()+ with wave.open(buffer, "wb") as output:+ output.setnchannels(1)+ output.setsampwidth(2)+ output.setframerate(24000)+ output.writeframes(b"\x00\x00" * 2400)+ mp3 = subprocess.run(+ [+ "ffmpeg",+ "-hide_banner",+ "-loglevel",+ "error",+ "-f",+ "wav",+ "-i",+ "pipe:0",+ "-f",+ "mp3",+ "pipe:1",+ ],+ input=buffer.getvalue(),+ capture_output=True,+ check=True,+ ).stdout+ path = _render_audio([(mp3, "audio/mpeg"), (mp3, "audio/mpeg")], tmp_path, "overview")+ assert path.suffix == ".mp3"+ decoded = subprocess.run(+ ["ffmpeg", "-hide_banner", "-loglevel", "error", "-i", str(path), "-f", "null", "-"],+ capture_output=True,+ check=False,+ )+ assert decoded.returncode == 0