Skip to content

利用 AI 大模型和自动化工作流,根据主题或关键词一键生成高清短视频。Generate HD short videos from a topic or keyword with an automated AI workflow.

feat(tts): add Chatterbox TTS provider (self-hosted, OpenAI-compatible)

Paulo C. Benetti023922ea2775
diff --git a/app/config/config.py b/app/config/config.py
index 72210eef..1ace0877 100644
--- a/app/config/config.py
+++ b/app/config/config.py
@@ -153,6 +153,7 @@ def save_config():
_cfg["azure"] = azure
_cfg["siliconflow"] = siliconflow
_cfg["elevenlabs"] = elevenlabs
+ _cfg["chatterbox"] = chatterbox
_cfg["ui"] = ui
f.write(toml.dumps(_cfg))
@@ -164,6 +165,7 @@ proxy = _cfg.get("proxy", {})
azure = _cfg.get("azure", {})
siliconflow = _cfg.get("siliconflow", {})
elevenlabs = _cfg.get("elevenlabs", {})
+chatterbox = _cfg.get("chatterbox", {})
ui = _cfg.get(
"ui",
{
diff --git a/app/services/voice.py b/app/services/voice.py
index 63780a06..190fa546 100644
--- a/app/services/voice.py
+++ b/app/services/voice.py
@@ -164,6 +164,29 @@ def get_elevenlabs_voices(api_key: str) -> list[str]:
return []
+def get_chatterbox_voices() -> list[str]:
+ """Return the configured Chatterbox voices.
+
+ Chatterbox is self-hosted, so there is no global voice catalog. Operators
+ list the voice names exposed by their server via ``[chatterbox] voices``
+ (a TOML array, or a comma-separated string). Each entry is normalised to
+ the ``chatterbox:<name>`` format used by the TTS dispatcher.
+ """
+ voices = config.chatterbox.get("voices", []) or []
+ if isinstance(voices, str):
+ voices = [v.strip() for v in voices.split(",") if v.strip()]
+ result = []
+ for v in voices:
+ v = str(v).strip()
+ if not v:
+ continue
+ result.append(v if v.startswith("chatterbox:") else f"chatterbox:{v}")
+ if not result:
+ # keep the dropdown usable even before any voice is configured
+ result = ["chatterbox:default-Female"]
+ return result
+
+
_AZURE_VOICES_DATA_FILE = os.path.join(
os.path.dirname(__file__), "data", "azure_voices.json"
)
@@ -229,6 +252,10 @@ def is_elevenlabs_voice(voice_name: str) -> bool:
return (voice_name or "").startswith("elevenlabs:")
+def is_chatterbox_voice(voice_name: str) -> bool:
+ return (voice_name or "").startswith("chatterbox:")
+
+
def is_no_voice(voice_name: str | None) -> bool:
"""
判断用户是否明确选择了“无配音”模式。
@@ -398,6 +425,19 @@ def tts(
else:
logger.error(f"Invalid elevenlabs voice name format: {voice_name}")
return None
+ elif is_chatterbox_voice(voice_name):
+ # 格式: chatterbox:<voice>,voice 可带显示用的 -Female/-Male 后缀
+ parts = voice_name.split(":", 1)
+ if len(parts) >= 2 and parts[1].strip():
+ chatterbox_voice = parts[1].strip()
+ if chatterbox_voice.endswith(("-Female", "-Male")):
+ chatterbox_voice = chatterbox_voice.rsplit("-", 1)[0]
+ return chatterbox_tts(
+ text, chatterbox_voice, voice_file, voice_rate, voice_volume
+ )
+ else:
+ logger.error(f"Invalid chatterbox voice name format: {voice_name}")
+ return None
return azure_tts_v1(text, voice_name, voice_rate, voice_file)
@@ -1269,6 +1309,89 @@ def elevenlabs_tts(
return None
+def chatterbox_tts(
+ text: str,
+ voice: str,
+ voice_file: str,
+ voice_rate: float = 1.0,
+ voice_volume: float = 1.0,
+ model_id: str = "",
+) -> Union[SubMaker, None]:
+ """Generate speech with a self-hosted Chatterbox TTS server.
+
+ Chatterbox (Resemble AI, MIT) is an open-source, locally hosted TTS model
+ with zero-shot voice cloning — a self-hostable alternative to ElevenLabs.
+ This talks to an OpenAI-compatible ``/audio/speech`` endpoint, so it works
+ with the common community servers (e.g. devnen/Chatterbox-TTS-Server,
+ travisvn/chatterbox-tts-api). Configure ``[chatterbox] base_url`` (and an
+ optional ``api_key``).
+
+ Like ElevenLabs, Chatterbox does not return word-level timestamps, so the
+ subtitle path falls back to the full-text SubMaker. For tighter subtitle
+ sync set ``subtitle_provider = "whisper"``.
+ """
+ text = (text or "").strip()
+ if not text:
+ logger.error("Chatterbox TTS text is empty")
+ return None
+
+ base_url = (config.chatterbox.get("base_url", "") or "").strip().rstrip("/")
+ if not base_url:
+ logger.error(
+ "Chatterbox base_url is not set, please configure [chatterbox] base_url in config.toml"
+ )
+ return None
+
+ api_key = config.chatterbox.get("api_key", "")
+ if not model_id:
+ model_id = config.chatterbox.get("model_id", "chatterbox") or "chatterbox"
+
+ url = f"{base_url}/audio/speech"
+ headers = {"Content-Type": "application/json"}
+ if api_key:
+ headers["Authorization"] = f"Bearer {api_key}"
+ payload = {
+ "model": model_id,
+ "input": text,
+ "voice": voice,
+ "response_format": "mp3",
+ # OpenAI speech API accepts speed 0.25-4.0; MoneyPrinterTurbo's rate is a
+ # 1.0-centred multiplier, so it maps directly (clamped to the valid range).
+ "speed": max(0.25, min(4.0, float(voice_rate or 1.0))),
+ }
+
+ for i in range(3):
+ try:
+ logger.info(f"start chatterbox tts, voice: {voice}, try: {i + 1}")
+ ensure_file_path_exists(voice_file)
+
+ response = requests.post(url, json=payload, headers=headers, timeout=120)
+ if response.status_code != 200:
+ logger.error(
+ f"chatterbox tts failed with status {response.status_code}: {response.text[:200]}"
+ )
+ continue
+
+ with open(voice_file, "wb") as f:
+ f.write(response.content)
+
+ audio_clip = AudioFileClip(voice_file)
+ audio_duration = audio_clip.duration
+ audio_clip.close()
+
+ sub_maker = ensure_legacy_submaker_fields(SubMaker())
+ logger.success(f"chatterbox tts succeeded: {voice_file}")
+ return populate_legacy_submaker_with_full_text(
+ sub_maker=sub_maker,
+ text=text,
+ audio_duration_seconds=audio_duration,
+ )
+ except Exception as e:
+ logger.error(f"chatterbox tts failed: {str(e)}")
+
+ return None
+
+
def _format_text(text: str) -> str:
"""
清理字幕对齐前的脚本文本。
diff --git a/config.example.toml b/config.example.toml
index 486cf5c1..f3bcb251 100644
--- a/config.example.toml
+++ b/config.example.toml
@@ -397,3 +397,19 @@ upload_post_auto_upload = false
# YouTube Shorts privacy status (options: "public", "unlisted", "private")
upload_post_youtube_privacy_status = "public"
+
+# Chatterbox TTS (self-hosted, OpenAI-compatible) ##########################
+# Chatterbox (https://github.com/resemble-ai/chatterbox, MIT) is an open-source,
+# locally hosted TTS model with zero-shot voice cloning — a self-hostable
+# alternative to ElevenLabs. Run a server that exposes an OpenAI-compatible
+# endpoint at POST {base_url}/audio/speech (e.g. devnen/Chatterbox-TTS-Server
+# or travisvn/chatterbox-tts-api), then set base_url below. In the WebUI pick
+# "Chatterbox TTS" as the TTS server, or via the API pass a voice_name like
+# "chatterbox:<voice>". Subtitles use the full-text fallback (no word-level
+# timestamps); set subtitle_provider = "whisper" for tighter sync.
+[chatterbox]
+# base_url = "http://localhost:4123/v1"
+# api_key = ""
+# model_id = "chatterbox"
+# Voices exposed by your server; shown in the WebUI dropdown as chatterbox:<name>
+# voices = ["Francisca-Female", "Antonio-Male"]
diff --git a/webui/Main.py b/webui/Main.py
index b9ed18d9..cddc26d1 100644
--- a/webui/Main.py
+++ b/webui/Main.py
@@ -964,6 +964,7 @@ with middle_panel:
("gemini-tts", "Google Gemini TTS"),
("mimo-tts", "Xiaomi MiMo TTS"),
("elevenlabs", "ElevenLabs TTS"),
+ ("chatterbox", "Chatterbox TTS"),
]
# 获取保存的TTS服务器,默认为v1
@@ -1016,6 +1017,9 @@ with middle_panel:
saved_elevenlabs_api_key
)
filtered_voices = st.session_state[cache_key]
+ elif selected_tts_server == "chatterbox":
+ # 自托管 Chatterbox 服务的预置音色(来自 [chatterbox] voices 配置)
+ filtered_voices = voice.get_chatterbox_voices()
else:
# 获取Azure的声音列表
all_voices = voice.get_all_azure_voices(filter_locals=None)
@@ -1038,6 +1042,9 @@ with middle_panel:
if voice.is_elevenlabs_voice(v):
parts = v.split(":", 2)
return parts[2] if len(parts) >= 3 else v
+ if voice.is_chatterbox_voice(v):
+ name = v.split(":", 1)[1] if ":" in v else v
+ return name.replace("-Female", "").replace("-Male", "")
return (
v.replace("Female", tr("Female"))
.replace("Male", tr("Male"))