DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.
Python ai-agentsai-tutorclawbotcli-tooldeepresearchinteractive-learninglearningmulti-agent-systemsrag
fix(voice): add native Gemini TTS adapter for generateContent speech
你的名字1cf108e07bb2
diff --git a/deeptutor/services/config/provider_runtime.py b/deeptutor/services/config/provider_runtime.pyindex ec3232915..8f04e735b 100644--- a/deeptutor/services/config/provider_runtime.py+++ b/deeptutor/services/config/provider_runtime.py@@ -350,6 +350,14 @@ TTS_PROVIDERS: dict[str, VoiceProviderSpec] = {default_api_base="https://api.openai.com/v1",default_model="gpt-4o-mini-tts",),+ # Gemini TTS only speaks the native generateContent audio API, never the+ # OpenAI-compatible /audio/speech endpoint, so it gets its own adapter.+ "gemini": VoiceProviderSpec(+ label="Google Gemini",+ default_api_base="https://generativelanguage.googleapis.com/v1beta",+ adapter="gemini",+ default_model="gemini-2.5-flash-preview-tts",+ ),"openrouter": VoiceProviderSpec(label="OpenRouter",default_api_base="https://openrouter.ai/api/v1",diff --git a/deeptutor/services/voice/adapters/__init__.py b/deeptutor/services/voice/adapters/__init__.pyindex 948177ca0..312cdaf4b 100644--- a/deeptutor/services/voice/adapters/__init__.py+++ b/deeptutor/services/voice/adapters/__init__.py@@ -12,6 +12,7 @@ from deeptutor.services.voice.adapters.dashscope import (DashScopeSTTAdapter,DashScopeTTSAdapter,)+from deeptutor.services.voice.adapters.gemini import GeminiTTSAdapterfrom deeptutor.services.voice.adapters.mimo import MiMoTTSAdapterfrom deeptutor.services.voice.adapters.minimax import MiniMaxTTSAdapterfrom deeptutor.services.voice.adapters.openai_compat import (@@ -29,6 +30,7 @@ TTS_ADAPTERS: dict[str, BaseTTSAdapter] = {"openai_compat": OpenAICompatTTSAdapter(),"openrouter_tts": OpenRouterTTSAdapter(),"dashscope": DashScopeTTSAdapter(),+ "gemini": GeminiTTSAdapter(),}STT_ADAPTERS: dict[str, BaseSTTAdapter] = {diff --git a/deeptutor/services/voice/adapters/gemini.py b/deeptutor/services/voice/adapters/gemini.pynew file mode 100644index 000000000..5b3a087a7--- /dev/null+++ b/deeptutor/services/voice/adapters/gemini.py@@ -0,0 +1,113 @@+"""Native Gemini speech synthesis through ``generateContent`` audio output.++Gemini TTS models are not reachable through the OpenAI-compatible+``/audio/speech`` endpoint; they only speak the native ``generateContent``+API with ``responseModalities: ["AUDIO"]`` and a prebuilt voice name.+Protocol: https://ai.google.dev/gemini-api/docs/speech-generation+"""++from __future__ import annotations++import base64+import binascii+from typing import Any+from urllib.parse import urlsplit++import httpx++from deeptutor.services.voice.adapters.openai_compat import _raise_for_provider+from deeptutor.services.voice.audio import pcm_to_wav+from deeptutor.services.voice.base import BaseTTSAdapter, VoiceProviderError+from deeptutor.services.voice.config import TTSConfig++# Gemini always answers with mono PCM16 at this rate, whatever format was asked.+GEMINI_TTS_SAMPLE_RATE = 24000+_GEMINI_API_HOST = "generativelanguage.googleapis.com"+++def _model_id(model: str) -> str:+ """Normalize discovery (``models/gemini-...``) and gateway (``google/...``) ids."""+ stripped = (model or "").strip().strip("/")+ for prefix in ("models/", "google/"):+ if stripped.startswith(prefix):+ stripped = stripped[len(prefix) :]+ if not stripped:+ raise VoiceProviderError("Enter a Gemini TTS model such as gemini-2.5-flash-preview-tts.")+ return stripped+++def _auth_headers(config: TTSConfig) -> dict[str, str]:+ """Prefer Google's native key header on the Google host, Bearer elsewhere."""+ headers = {"Content-Type": "application/json"}+ if config.api_key and config.api_key != "***":+ if urlsplit(config.base_url).hostname == _GEMINI_API_HOST:+ headers["x-goog-api-key"] = config.api_key+ else:+ headers["Authorization"] = f"Bearer {config.api_key}"+ return headers+++class GeminiTTSAdapter(BaseTTSAdapter):+ """Synthesize Google prebuilt voices through native ``generateContent``."""++ async def synthesize(self, text: str, config: TTSConfig) -> tuple[bytes, str]:+ """Return WAV-wrapped (or raw) PCM16 24 kHz audio from the audio modality."""+ model_id = _model_id(config.model)+ voice = (config.voice or "").strip()+ if not voice:+ raise VoiceProviderError(+ "Choose a Google prebuilt voice name such as Kore or Puck for Gemini TTS."+ )+ audio_format = (config.response_format or "wav").strip().lower()+ if audio_format == "pcm16":+ audio_format = "pcm"+ if audio_format not in {"wav", "pcm"}:+ raise VoiceProviderError("Gemini TTS supports WAV or PCM in this adapter.")+ if config.speed is not None:+ raise VoiceProviderError("Gemini TTS has no speed parameter; use voice instructions.")+ # Style control is prompt-level: the documented way to steer delivery is+ # to lead the input text with a natural-language direction.+ spoken = f"{config.instructions}: {text}" if config.instructions else text+ payload: dict[str, Any] = {+ "contents": [{"parts": [{"text": spoken}]}],+ "generationConfig": {+ "responseModalities": ["AUDIO"],+ "speechConfig": {"voiceConfig": {"prebuiltVoiceConfig": {"voiceName": voice}}},+ },+ }+ base = (config.base_url or "").strip().rstrip("/")+ if not base:+ raise VoiceProviderError("No endpoint URL configured for this provider.")+ # A connection saved with Gemini's OpenAI-compatible URL still reaches+ # the native API once the compatibility suffix is dropped.+ if base.endswith("/openai"):+ base = base[: -len("/openai")]+ url = f"{base}/models/{model_id}:generateContent"+ headers = {**_auth_headers(config), **(config.extra_headers or {})}+ try:+ async with httpx.AsyncClient(timeout=config.request_timeout) as client:+ response = await client.post(url, headers=headers, json=payload)+ except httpx.HTTPError as exc:+ raise VoiceProviderError("Gemini TTS connection failed or timed out.") from exc+ _raise_for_provider(response, "Gemini TTS synthesis")+ try:+ candidate = response.json()["candidates"][0]+ encoded_parts = [+ part["inlineData"]["data"]+ for part in candidate["content"]["parts"]+ if isinstance(part.get("inlineData"), dict)+ and isinstance(part["inlineData"].get("data"), str)+ ]+ # Each inlineData value is independently padded base64; decode parts+ # separately instead of concatenating the padded strings.+ audio = b"".join(base64.b64decode(part, validate=True) for part in encoded_parts)+ except (ValueError, KeyError, IndexError, TypeError, binascii.Error) as exc:+ # Provider bodies can echo the prompt; never surface them raw.+ raise VoiceProviderError(+ "Gemini TTS returned no audio; check the model, voice and API key."+ ) from exc+ if not audio:+ raise VoiceProviderError("Gemini TTS returned empty audio.")+ if audio_format == "pcm":+ return audio, f"audio/pcm;rate={GEMINI_TTS_SAMPLE_RATE};channels=1"+ return pcm_to_wav(audio, GEMINI_TTS_SAMPLE_RATE), "audio/wav"diff --git a/deeptutor/services/voice/options.py b/deeptutor/services/voice/options.pyindex f659c10af..38c848947 100644--- a/deeptutor/services/voice/options.py+++ b/deeptutor/services/voice/options.py@@ -136,6 +136,23 @@ def _tts_models(provider: str) -> tuple[list[dict], str]:)for m in audio_models], "https://help.aliyun.com/zh/model-studio/qwen-tts-voice-list"+ if provider == "gemini":+ return [+ preset(+ model,+ formats=["wav", "pcm"],+ instructions=True,+ configuration_note=(+ "Gemini TTS outputs PCM audio served as WAV. Voices are Google "+ "prebuilt names such as Kore or Puck."+ ),+ )+ for model in [+ "gemini-2.5-flash-preview-tts",+ "gemini-2.5-pro-preview-tts",+ "gemini-3.1-flash-tts-preview",+ ]+ ], "https://ai.google.dev/gemini-api/docs/speech-generation"if provider == "groq":return [preset(@@ -182,6 +199,8 @@ def voice_options(provider: str, service: str) -> dict:fallback = preset("", formats=["wav"])elif provider == "xiaomi_mimo":fallback = preset("", formats=["wav", "pcm16"], instructions=True)+ elif provider == "gemini":+ fallback = preset("", formats=["wav", "pcm"], instructions=True)else:ids = {"openai": ["gpt-4o-mini-transcribe", "gpt-4o-transcribe", "whisper-1"],diff --git a/tests/services/test_gemini_voice.py b/tests/services/test_gemini_voice.pynew file mode 100644index 000000000..b32cbae67--- /dev/null+++ b/tests/services/test_gemini_voice.py@@ -0,0 +1,210 @@+"""Regression checks for Gemini's native generateContent speech protocol."""++import base64+import io+import json+from pathlib import Path+import tempfile+from typing import Any+import unittest+from unittest.mock import patch+import wave++import httpx++from deeptutor.services.config.model_catalog import ModelCatalogService+from deeptutor.services.config.provider_runtime import resolve_tts_runtime_config+from deeptutor.services.voice.adapters import get_tts_adapter+from deeptutor.services.voice.adapters.gemini import GeminiTTSAdapter+from deeptutor.services.voice.base import VoiceProviderError, VoiceProviderHTTPError+from deeptutor.services.voice.config import TTSConfig+from deeptutor.services.voice.options import voice_model_options++API_ROOT = "https://generativelanguage.googleapis.com/v1beta"+++def gemini_response(chunks: list[bytes], **extra: Any) -> dict[str, Any]:+ parts = [+ {+ "inlineData": {+ "mimeType": "audio/L16;codec=pcm;rate=24000",+ "data": base64.b64encode(chunk).decode("ascii"),+ }+ }+ for chunk in chunks+ ]+ return {"candidates": [{"content": {"parts": parts}}], **extra}+++class GeminiTTSTests(unittest.IsolatedAsyncioTestCase):+ """Exercise the Gemini wire protocol without making external requests."""++ async def invoke(+ self, config: TTSConfig, response: dict[str, Any], status: int = 200+ ) -> tuple[tuple[bytes, str], list[httpx.Request]]:+ requests: list[httpx.Request] = []++ def handler(request: httpx.Request) -> httpx.Response:+ requests.append(request)+ return httpx.Response(status, json=response)++ client_class = httpx.AsyncClient+ with patch(+ "deeptutor.services.voice.adapters.gemini.httpx.AsyncClient",+ side_effect=lambda **kwargs: client_class(+ transport=httpx.MockTransport(handler), **kwargs+ ),+ ):+ result = await GeminiTTSAdapter().synthesize("你好", config)+ return result, requests++ def config(self, **kwargs: Any) -> TTSConfig:+ defaults: dict[str, Any] = {+ "model": "gemini-2.5-flash-preview-tts",+ "base_url": API_ROOT,+ "api_key": "test-key",+ "voice": "Kore",+ "response_format": "wav",+ }+ return TTSConfig(**{**defaults, **kwargs})++ async def test_request_shape_and_wav_wrap(self) -> None:+ pcm = b"\x00\x01" * 12+ result, requests = await self.invoke(self.config(), gemini_response([pcm]))+ self.assertEqual(len(requests), 1)+ request = requests[0]+ self.assertEqual(+ str(request.url), f"{API_ROOT}/models/gemini-2.5-flash-preview-tts:generateContent"+ )+ self.assertNotIn("Authorization", request.headers)+ self.assertEqual(request.headers["x-goog-api-key"], "test-key")+ self.assertEqual(+ json.loads(request.content),+ {+ "contents": [{"parts": [{"text": "你好"}]}],+ "generationConfig": {+ "responseModalities": ["AUDIO"],+ "speechConfig": {"voiceConfig": {"prebuiltVoiceConfig": {"voiceName": "Kore"}}},+ },+ },+ )+ audio, content_type = result+ self.assertEqual(content_type, "audio/wav")+ with wave.open(io.BytesIO(audio), "rb") as wav:+ self.assertEqual(wav.getnchannels(), 1)+ self.assertEqual(wav.getsampwidth(), 2)+ self.assertEqual(wav.getframerate(), 24000)+ self.assertEqual(wav.readframes(wav.getnframes()), pcm)++ async def test_model_prefixes_and_openai_suffix_are_normalized(self) -> None:+ config = self.config(+ model="models/gemini-3.1-flash-tts-preview",+ base_url=f"{API_ROOT}/openai/",+ )+ _, requests = await self.invoke(config, gemini_response([b"\x00\x00"]))+ self.assertEqual(+ str(requests[0].url),+ f"{API_ROOT}/models/gemini-3.1-flash-tts-preview:generateContent",+ )++ async def test_gateway_base_url_uses_bearer_auth(self) -> None:+ config = self.config(base_url="https://gateway.example.com/gemini")+ _, requests = await self.invoke(config, gemini_response([b"\x00\x00"]))+ self.assertNotIn("x-goog-api-key", requests[0].headers)+ self.assertEqual(requests[0].headers["Authorization"], "Bearer test-key")++ async def test_instructions_prefix_and_pcm_passthrough(self) -> None:+ pcm = b"\x01\x02"+ config = self.config(instructions="Say cheerfully", response_format="pcm16", speed=None)+ result, requests = await self.invoke(config, gemini_response([pcm]))+ self.assertEqual(result, (pcm, "audio/pcm;rate=24000;channels=1"))+ self.assertEqual(+ json.loads(requests[0].content)["contents"][0]["parts"][0]["text"],+ "Say cheerfully: 你好",+ )++ async def test_multi_part_audio_is_concatenated(self) -> None:+ chunks = [b"\x00\x01", b"\x02\x03", b"\x04\x05"]+ result, _ = await self.invoke(self.config(), gemini_response(chunks))+ with wave.open(io.BytesIO(result[0]), "rb") as wav:+ self.assertEqual(wav.readframes(wav.getnframes()), b"".join(chunks))++ async def test_bad_audio_and_http_failure(self) -> None:+ for response in [+ {},+ {"candidates": []},+ {"candidates": [{"content": {"parts": [{"text": "no audio"}]}}]},+ gemini_response([b""]),+ {"candidates": [{"content": {"parts": [{"inlineData": {"data": "not-base64!"}}]}}]},+ ]:+ with self.subTest(response=response), self.assertRaises(VoiceProviderError):+ await self.invoke(self.config(), response)+ with self.assertRaises(VoiceProviderHTTPError) as raised:+ await self.invoke(self.config(), {"error": {"message": "unauthorized"}}, 401)+ self.assertEqual(raised.exception.status_code, 401)++ async def test_unsupported_parameters_fail_before_request(self) -> None:+ for field, value in (+ ("response_format", "mp3"),+ ("speed", 1.2),+ ("voice", ""),+ ("base_url", ""),+ ):+ config = self.config()+ setattr(config, field, value)+ with self.subTest(field=field):+ with patch("deeptutor.services.voice.adapters.gemini.httpx.AsyncClient") as client:+ with self.assertRaises(VoiceProviderError):+ await GeminiTTSAdapter().synthesize("你好", config)+ client.assert_not_called()++ def test_linked_catalog_selects_native_adapter_and_defaults(self) -> None:+ ref = {"connection_id": "gemini-conn", "binding": "gemini"}+ catalog = {+ "connections": [+ {+ "id": "gemini-conn",+ "provider": "gemini",+ "source_service": "llm",+ "base_url": "",+ "api_key": "test-key",+ }+ ],+ "services": {+ "tts": {+ "active_profile_id": "profile",+ "active_model_id": "model",+ "profiles": [+ {+ "id": "profile",+ "provider_ref": ref,+ "models": [+ {+ "id": "model",+ "model": "models/gemini-2.5-flash-preview-tts",+ "voice": "Kore",+ "provider_ref": ref,+ }+ ],+ }+ ],+ }+ },+ }+ with tempfile.TemporaryDirectory() as directory:+ service = ModelCatalogService(Path(directory) / "catalog.json")+ config = resolve_tts_runtime_config(catalog, service=service)+ self.assertEqual(config.provider_name, "gemini")+ self.assertEqual(config.adapter, "gemini")+ self.assertEqual(config.base_url, API_ROOT)+ self.assertEqual(config.api_key, "test-key")+ self.assertEqual(config.voice, "Kore")+ self.assertEqual(config.response_format, "wav")+ self.assertIsInstance(get_tts_adapter(config.adapter), GeminiTTSAdapter)+ options = voice_model_options("gemini", "tts", config.model)+ self.assertIn("wav", options["formats"])+ self.assertTrue(options["instructions"])+++if __name__ == "__main__":+ unittest.main()