Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

fix(voice): add native Gemini TTS adapter for generateContent speech

你的名字1cf108e07bb2
diff --git a/deeptutor/services/config/provider_runtime.py b/deeptutor/services/config/provider_runtime.py
index ec3232915..8f04e735b 100644
--- a/deeptutor/services/config/provider_runtime.py
+++ b/deeptutor/services/config/provider_runtime.py
@@ -350,6 +350,14 @@ TTS_PROVIDERS: dict[str, VoiceProviderSpec] = {
default_api_base="https://api.openai.com/v1",
default_model="gpt-4o-mini-tts",
),
+ # Gemini TTS only speaks the native generateContent audio API, never the
+ # OpenAI-compatible /audio/speech endpoint, so it gets its own adapter.
+ "gemini": VoiceProviderSpec(
+ label="Google Gemini",
+ default_api_base="https://generativelanguage.googleapis.com/v1beta",
+ adapter="gemini",
+ default_model="gemini-2.5-flash-preview-tts",
+ ),
"openrouter": VoiceProviderSpec(
label="OpenRouter",
default_api_base="https://openrouter.ai/api/v1",
diff --git a/deeptutor/services/voice/adapters/__init__.py b/deeptutor/services/voice/adapters/__init__.py
index 948177ca0..312cdaf4b 100644
--- a/deeptutor/services/voice/adapters/__init__.py
+++ b/deeptutor/services/voice/adapters/__init__.py
@@ -12,6 +12,7 @@ from deeptutor.services.voice.adapters.dashscope import (
DashScopeSTTAdapter,
DashScopeTTSAdapter,
)
+from deeptutor.services.voice.adapters.gemini import GeminiTTSAdapter
from deeptutor.services.voice.adapters.mimo import MiMoTTSAdapter
from deeptutor.services.voice.adapters.minimax import MiniMaxTTSAdapter
from deeptutor.services.voice.adapters.openai_compat import (
@@ -29,6 +30,7 @@ TTS_ADAPTERS: dict[str, BaseTTSAdapter] = {
"openai_compat": OpenAICompatTTSAdapter(),
"openrouter_tts": OpenRouterTTSAdapter(),
"dashscope": DashScopeTTSAdapter(),
+ "gemini": GeminiTTSAdapter(),
}
STT_ADAPTERS: dict[str, BaseSTTAdapter] = {
diff --git a/deeptutor/services/voice/adapters/gemini.py b/deeptutor/services/voice/adapters/gemini.py
new file mode 100644
index 000000000..5b3a087a7
--- /dev/null
+++ b/deeptutor/services/voice/adapters/gemini.py
@@ -0,0 +1,113 @@
+"""Native Gemini speech synthesis through ``generateContent`` audio output.
+
+Gemini TTS models are not reachable through the OpenAI-compatible
+``/audio/speech`` endpoint; they only speak the native ``generateContent``
+API with ``responseModalities: ["AUDIO"]`` and a prebuilt voice name.
+Protocol: https://ai.google.dev/gemini-api/docs/speech-generation
+"""
+
+from __future__ import annotations
+
+import base64
+import binascii
+from typing import Any
+from urllib.parse import urlsplit
+
+import httpx
+
+from deeptutor.services.voice.adapters.openai_compat import _raise_for_provider
+from deeptutor.services.voice.audio import pcm_to_wav
+from deeptutor.services.voice.base import BaseTTSAdapter, VoiceProviderError
+from deeptutor.services.voice.config import TTSConfig
+
+# Gemini always answers with mono PCM16 at this rate, whatever format was asked.
+GEMINI_TTS_SAMPLE_RATE = 24000
+_GEMINI_API_HOST = "generativelanguage.googleapis.com"
+
+
+def _model_id(model: str) -> str:
+ """Normalize discovery (``models/gemini-...``) and gateway (``google/...``) ids."""
+ stripped = (model or "").strip().strip("/")
+ for prefix in ("models/", "google/"):
+ if stripped.startswith(prefix):
+ stripped = stripped[len(prefix) :]
+ if not stripped:
+ raise VoiceProviderError("Enter a Gemini TTS model such as gemini-2.5-flash-preview-tts.")
+ return stripped
+
+
+def _auth_headers(config: TTSConfig) -> dict[str, str]:
+ """Prefer Google's native key header on the Google host, Bearer elsewhere."""
+ headers = {"Content-Type": "application/json"}
+ if config.api_key and config.api_key != "***":
+ if urlsplit(config.base_url).hostname == _GEMINI_API_HOST:
+ headers["x-goog-api-key"] = config.api_key
+ else:
+ headers["Authorization"] = f"Bearer {config.api_key}"
+ return headers
+
+
+class GeminiTTSAdapter(BaseTTSAdapter):
+ """Synthesize Google prebuilt voices through native ``generateContent``."""
+
+ async def synthesize(self, text: str, config: TTSConfig) -> tuple[bytes, str]:
+ """Return WAV-wrapped (or raw) PCM16 24 kHz audio from the audio modality."""
+ model_id = _model_id(config.model)
+ voice = (config.voice or "").strip()
+ if not voice:
+ raise VoiceProviderError(
+ "Choose a Google prebuilt voice name such as Kore or Puck for Gemini TTS."
+ )
+ audio_format = (config.response_format or "wav").strip().lower()
+ if audio_format == "pcm16":
+ audio_format = "pcm"
+ if audio_format not in {"wav", "pcm"}:
+ raise VoiceProviderError("Gemini TTS supports WAV or PCM in this adapter.")
+ if config.speed is not None:
+ raise VoiceProviderError("Gemini TTS has no speed parameter; use voice instructions.")
+ # Style control is prompt-level: the documented way to steer delivery is
+ # to lead the input text with a natural-language direction.
+ spoken = f"{config.instructions}: {text}" if config.instructions else text
+ payload: dict[str, Any] = {
+ "contents": [{"parts": [{"text": spoken}]}],
+ "generationConfig": {
+ "responseModalities": ["AUDIO"],
+ "speechConfig": {"voiceConfig": {"prebuiltVoiceConfig": {"voiceName": voice}}},
+ },
+ }
+ base = (config.base_url or "").strip().rstrip("/")
+ if not base:
+ raise VoiceProviderError("No endpoint URL configured for this provider.")
+ # A connection saved with Gemini's OpenAI-compatible URL still reaches
+ # the native API once the compatibility suffix is dropped.
+ if base.endswith("/openai"):
+ base = base[: -len("/openai")]
+ url = f"{base}/models/{model_id}:generateContent"
+ headers = {**_auth_headers(config), **(config.extra_headers or {})}
+ try:
+ async with httpx.AsyncClient(timeout=config.request_timeout) as client:
+ response = await client.post(url, headers=headers, json=payload)
+ except httpx.HTTPError as exc:
+ raise VoiceProviderError("Gemini TTS connection failed or timed out.") from exc
+ _raise_for_provider(response, "Gemini TTS synthesis")
+ try:
+ candidate = response.json()["candidates"][0]
+ encoded_parts = [
+ part["inlineData"]["data"]
+ for part in candidate["content"]["parts"]
+ if isinstance(part.get("inlineData"), dict)
+ and isinstance(part["inlineData"].get("data"), str)
+ ]
+ # Each inlineData value is independently padded base64; decode parts
+ # separately instead of concatenating the padded strings.
+ audio = b"".join(base64.b64decode(part, validate=True) for part in encoded_parts)
+ except (ValueError, KeyError, IndexError, TypeError, binascii.Error) as exc:
+ # Provider bodies can echo the prompt; never surface them raw.
+ raise VoiceProviderError(
+ "Gemini TTS returned no audio; check the model, voice and API key."
+ ) from exc
+ if not audio:
+ raise VoiceProviderError("Gemini TTS returned empty audio.")
+ if audio_format == "pcm":
+ return audio, f"audio/pcm;rate={GEMINI_TTS_SAMPLE_RATE};channels=1"
+ return pcm_to_wav(audio, GEMINI_TTS_SAMPLE_RATE), "audio/wav"
diff --git a/deeptutor/services/voice/options.py b/deeptutor/services/voice/options.py
index f659c10af..38c848947 100644
--- a/deeptutor/services/voice/options.py
+++ b/deeptutor/services/voice/options.py
@@ -136,6 +136,23 @@ def _tts_models(provider: str) -> tuple[list[dict], str]:
)
for m in audio_models
], "https://help.aliyun.com/zh/model-studio/qwen-tts-voice-list"
+ if provider == "gemini":
+ return [
+ preset(
+ model,
+ formats=["wav", "pcm"],
+ instructions=True,
+ configuration_note=(
+ "Gemini TTS outputs PCM audio served as WAV. Voices are Google "
+ "prebuilt names such as Kore or Puck."
+ ),
+ )
+ for model in [
+ "gemini-2.5-flash-preview-tts",
+ "gemini-2.5-pro-preview-tts",
+ "gemini-3.1-flash-tts-preview",
+ ]
+ ], "https://ai.google.dev/gemini-api/docs/speech-generation"
if provider == "groq":
return [
preset(
@@ -182,6 +199,8 @@ def voice_options(provider: str, service: str) -> dict:
fallback = preset("", formats=["wav"])
elif provider == "xiaomi_mimo":
fallback = preset("", formats=["wav", "pcm16"], instructions=True)
+ elif provider == "gemini":
+ fallback = preset("", formats=["wav", "pcm"], instructions=True)
else:
ids = {
"openai": ["gpt-4o-mini-transcribe", "gpt-4o-transcribe", "whisper-1"],
diff --git a/tests/services/test_gemini_voice.py b/tests/services/test_gemini_voice.py
new file mode 100644
index 000000000..b32cbae67
--- /dev/null
+++ b/tests/services/test_gemini_voice.py
@@ -0,0 +1,210 @@
+"""Regression checks for Gemini's native generateContent speech protocol."""
+
+import base64
+import io
+import json
+from pathlib import Path
+import tempfile
+from typing import Any
+import unittest
+from unittest.mock import patch
+import wave
+
+import httpx
+
+from deeptutor.services.config.model_catalog import ModelCatalogService
+from deeptutor.services.config.provider_runtime import resolve_tts_runtime_config
+from deeptutor.services.voice.adapters import get_tts_adapter
+from deeptutor.services.voice.adapters.gemini import GeminiTTSAdapter
+from deeptutor.services.voice.base import VoiceProviderError, VoiceProviderHTTPError
+from deeptutor.services.voice.config import TTSConfig
+from deeptutor.services.voice.options import voice_model_options
+
+API_ROOT = "https://generativelanguage.googleapis.com/v1beta"
+
+
+def gemini_response(chunks: list[bytes], **extra: Any) -> dict[str, Any]:
+ parts = [
+ {
+ "inlineData": {
+ "mimeType": "audio/L16;codec=pcm;rate=24000",
+ "data": base64.b64encode(chunk).decode("ascii"),
+ }
+ }
+ for chunk in chunks
+ ]
+ return {"candidates": [{"content": {"parts": parts}}], **extra}
+
+
+class GeminiTTSTests(unittest.IsolatedAsyncioTestCase):
+ """Exercise the Gemini wire protocol without making external requests."""
+
+ async def invoke(
+ self, config: TTSConfig, response: dict[str, Any], status: int = 200
+ ) -> tuple[tuple[bytes, str], list[httpx.Request]]:
+ requests: list[httpx.Request] = []
+
+ def handler(request: httpx.Request) -> httpx.Response:
+ requests.append(request)
+ return httpx.Response(status, json=response)
+
+ client_class = httpx.AsyncClient
+ with patch(
+ "deeptutor.services.voice.adapters.gemini.httpx.AsyncClient",
+ side_effect=lambda **kwargs: client_class(
+ transport=httpx.MockTransport(handler), **kwargs
+ ),
+ ):
+ result = await GeminiTTSAdapter().synthesize("你好", config)
+ return result, requests
+
+ def config(self, **kwargs: Any) -> TTSConfig:
+ defaults: dict[str, Any] = {
+ "model": "gemini-2.5-flash-preview-tts",
+ "base_url": API_ROOT,
+ "api_key": "test-key",
+ "voice": "Kore",
+ "response_format": "wav",
+ }
+ return TTSConfig(**{**defaults, **kwargs})
+
+ async def test_request_shape_and_wav_wrap(self) -> None:
+ pcm = b"\x00\x01" * 12
+ result, requests = await self.invoke(self.config(), gemini_response([pcm]))
+ self.assertEqual(len(requests), 1)
+ request = requests[0]
+ self.assertEqual(
+ str(request.url), f"{API_ROOT}/models/gemini-2.5-flash-preview-tts:generateContent"
+ )
+ self.assertNotIn("Authorization", request.headers)
+ self.assertEqual(request.headers["x-goog-api-key"], "test-key")
+ self.assertEqual(
+ json.loads(request.content),
+ {
+ "contents": [{"parts": [{"text": "你好"}]}],
+ "generationConfig": {
+ "responseModalities": ["AUDIO"],
+ "speechConfig": {"voiceConfig": {"prebuiltVoiceConfig": {"voiceName": "Kore"}}},
+ },
+ },
+ )
+ audio, content_type = result
+ self.assertEqual(content_type, "audio/wav")
+ with wave.open(io.BytesIO(audio), "rb") as wav:
+ self.assertEqual(wav.getnchannels(), 1)
+ self.assertEqual(wav.getsampwidth(), 2)
+ self.assertEqual(wav.getframerate(), 24000)
+ self.assertEqual(wav.readframes(wav.getnframes()), pcm)
+
+ async def test_model_prefixes_and_openai_suffix_are_normalized(self) -> None:
+ config = self.config(
+ model="models/gemini-3.1-flash-tts-preview",
+ base_url=f"{API_ROOT}/openai/",
+ )
+ _, requests = await self.invoke(config, gemini_response([b"\x00\x00"]))
+ self.assertEqual(
+ str(requests[0].url),
+ f"{API_ROOT}/models/gemini-3.1-flash-tts-preview:generateContent",
+ )
+
+ async def test_gateway_base_url_uses_bearer_auth(self) -> None:
+ config = self.config(base_url="https://gateway.example.com/gemini")
+ _, requests = await self.invoke(config, gemini_response([b"\x00\x00"]))
+ self.assertNotIn("x-goog-api-key", requests[0].headers)
+ self.assertEqual(requests[0].headers["Authorization"], "Bearer test-key")
+
+ async def test_instructions_prefix_and_pcm_passthrough(self) -> None:
+ pcm = b"\x01\x02"
+ config = self.config(instructions="Say cheerfully", response_format="pcm16", speed=None)
+ result, requests = await self.invoke(config, gemini_response([pcm]))
+ self.assertEqual(result, (pcm, "audio/pcm;rate=24000;channels=1"))
+ self.assertEqual(
+ json.loads(requests[0].content)["contents"][0]["parts"][0]["text"],
+ "Say cheerfully: 你好",
+ )
+
+ async def test_multi_part_audio_is_concatenated(self) -> None:
+ chunks = [b"\x00\x01", b"\x02\x03", b"\x04\x05"]
+ result, _ = await self.invoke(self.config(), gemini_response(chunks))
+ with wave.open(io.BytesIO(result[0]), "rb") as wav:
+ self.assertEqual(wav.readframes(wav.getnframes()), b"".join(chunks))
+
+ async def test_bad_audio_and_http_failure(self) -> None:
+ for response in [
+ {},
+ {"candidates": []},
+ {"candidates": [{"content": {"parts": [{"text": "no audio"}]}}]},
+ gemini_response([b""]),
+ {"candidates": [{"content": {"parts": [{"inlineData": {"data": "not-base64!"}}]}}]},
+ ]:
+ with self.subTest(response=response), self.assertRaises(VoiceProviderError):
+ await self.invoke(self.config(), response)
+ with self.assertRaises(VoiceProviderHTTPError) as raised:
+ await self.invoke(self.config(), {"error": {"message": "unauthorized"}}, 401)
+ self.assertEqual(raised.exception.status_code, 401)
+
+ async def test_unsupported_parameters_fail_before_request(self) -> None:
+ for field, value in (
+ ("response_format", "mp3"),
+ ("speed", 1.2),
+ ("voice", ""),
+ ("base_url", ""),
+ ):
+ config = self.config()
+ setattr(config, field, value)
+ with self.subTest(field=field):
+ with patch("deeptutor.services.voice.adapters.gemini.httpx.AsyncClient") as client:
+ with self.assertRaises(VoiceProviderError):
+ await GeminiTTSAdapter().synthesize("你好", config)
+ client.assert_not_called()
+
+ def test_linked_catalog_selects_native_adapter_and_defaults(self) -> None:
+ ref = {"connection_id": "gemini-conn", "binding": "gemini"}
+ catalog = {
+ "connections": [
+ {
+ "id": "gemini-conn",
+ "provider": "gemini",
+ "source_service": "llm",
+ "base_url": "",
+ "api_key": "test-key",
+ }
+ ],
+ "services": {
+ "tts": {
+ "active_profile_id": "profile",
+ "active_model_id": "model",
+ "profiles": [
+ {
+ "id": "profile",
+ "provider_ref": ref,
+ "models": [
+ {
+ "id": "model",
+ "model": "models/gemini-2.5-flash-preview-tts",
+ "voice": "Kore",
+ "provider_ref": ref,
+ }
+ ],
+ }
+ ],
+ }
+ },
+ }
+ with tempfile.TemporaryDirectory() as directory:
+ service = ModelCatalogService(Path(directory) / "catalog.json")
+ config = resolve_tts_runtime_config(catalog, service=service)
+ self.assertEqual(config.provider_name, "gemini")
+ self.assertEqual(config.adapter, "gemini")
+ self.assertEqual(config.base_url, API_ROOT)
+ self.assertEqual(config.api_key, "test-key")
+ self.assertEqual(config.voice, "Kore")
+ self.assertEqual(config.response_format, "wav")
+ self.assertIsInstance(get_tts_adapter(config.adapter), GeminiTTSAdapter)
+ options = voice_model_options("gemini", "tts", config.model)
+ self.assertIn("wav", options["formats"])
+ self.assertTrue(options["instructions"])
+
+
+if __name__ == "__main__":
+ unittest.main()