Skip to content

利用 AI 大模型和自动化工作流,根据主题或关键词一键生成高清短视频。Generate HD short videos from a topic or keyword with an automated AI workflow.

fix(tts): address all maintainer review feedback on Fish Audio integration

Hariom paswan375a45806f2f
diff --git a/app/services/voice.py b/app/services/voice.py
index d4d46f2a..7266fa14 100644
--- a/app/services/voice.py
+++ b/app/services/voice.py
@@ -222,12 +222,31 @@ def get_chatterbox_voices() -> list[str]:
def get_fish_audio_voices() -> list[str]:
- """获取 Fish Audio 预置声音与模型列表。"""
- return [
- "fish_audio:s2.1-pro-free:Free-Developer",
- "fish_audio:s2.1-pro:Pro-Voice",
- "fish_audio:s2-pro:Pro-Voice",
- ]
+ """Return configured Fish Audio voices.
+
+ Each entry follows the format ``fish_audio:<reference_id>:<display_name>``.
+ When ``reference_id`` is "default", Fish Audio's built-in default voice is
+ used (no ``reference_id`` is sent in the API request). Operators can list
+ additional public or cloned voices via ``[fish_audio] voices`` in the
+ config file.
+ """
+ result = ["fish_audio:default:Default Voice"]
+ voices = config.fish_audio.get("voices", []) or []
+ if isinstance(voices, str):
+ voices = [v.strip() for v in voices.split(",") if v.strip()]
+ for entry in voices:
+ entry = str(entry).strip()
+ if not entry:
+ continue
+ if entry.startswith("fish_audio:"):
+ result.append(entry)
+ elif ":" in entry:
+ # "<reference_id>:<display_name>"
+ result.append(f"fish_audio:{entry}")
+ else:
+ # bare reference_id
+ result.append(f"fish_audio:{entry}:{entry}")
+ return result
_AZURE_VOICES_DATA_FILE = os.path.join(
@@ -523,8 +542,10 @@ def tts(
return None
elif is_fish_audio_voice(voice_name):
parts = voice_name.split(":")
- model_name = parts[1] if len(parts) >= 2 else "s2.1-pro-free"
- return fish_audio_tts(text, model_name, voice_file, voice_rate, voice_volume)
+ reference_id = parts[1] if len(parts) >= 2 else "default"
+ if reference_id == "default":
+ reference_id = None
+ return fish_audio_tts(text, voice_file, voice_rate, voice_volume, reference_id=reference_id)
return azure_tts_v1(text, voice_name, voice_rate, voice_file)
@@ -1764,14 +1785,28 @@ def chatterbox_tts(
return None
+# Fish Audio supported models.
+FISH_AUDIO_MODELS = ("s2.1-pro-free", "s2.1-pro", "s2-pro")
+FISH_AUDIO_DEFAULT_MODEL = "s2.1-pro-free"
+
+
def fish_audio_tts(
text: str,
- model_name: str,
voice_file: str,
voice_rate: float = 1.0,
voice_volume: float = 1.0,
+ reference_id: str | None = None,
) -> Union[SubMaker, None]:
- """Generate speech using Fish Audio TTS API."""
+ """Generate speech using Fish Audio TTS API.
+
+ The model is read from ``config.fish_audio["model"]`` (single source of
+ truth). ``reference_id`` selects a public or cloned voice; when *None*
+ Fish Audio's built-in default voice is used.
+
+ ``voice_rate`` is mapped to the ``prosody.speed`` field (0.5–2.0) and
+ ``voice_volume`` is converted from a linear multiplier to dB for the
+ ``prosody.volume`` field (-20.0–20.0 dB).
+ """
text = (text or "").strip()
if not text:
logger.error("Fish Audio TTS text is empty")
@@ -1780,36 +1815,95 @@ def fish_audio_tts(
api_key = get_fish_audio_api_key()
if not api_key:
logger.error(
- "Fish Audio API key is not set. Please set it in config.toml [fish_audio] or FISH_API_KEY environment variable."
+ "Fish Audio API key is not set. Please set it in config.toml "
+ "[fish_audio] or FISH_API_KEY environment variable."
)
return None
+ model_name = str(
+ config.fish_audio.get("model", FISH_AUDIO_DEFAULT_MODEL)
+ or FISH_AUDIO_DEFAULT_MODEL
+ ).strip()
+ if model_name not in FISH_AUDIO_MODELS:
+ logger.warning(
+ f"Unknown Fish Audio model '{model_name}', falling back to "
+ f"'{FISH_AUDIO_DEFAULT_MODEL}'"
+ )
+ model_name = FISH_AUDIO_DEFAULT_MODEL
+
+ # Map voice_rate → prosody.speed (0.5–2.0)
+ try:
+ speed = max(0.5, min(2.0, float(voice_rate or 1.0)))
+ except (TypeError, ValueError):
+ speed = 1.0
+
+ # Map voice_volume (linear multiplier) → prosody.volume (dB, -20–20).
+ # A multiplier of 1.0 → 0 dB; 0.1 → -20 dB; 2.0 → +6 dB.
+ import math
+ try:
+ vol = float(voice_volume or 1.0)
+ if vol <= 0:
+ volume_db = -20.0
+ else:
+ volume_db = max(-20.0, min(20.0, 20.0 * math.log10(vol)))
+ except (TypeError, ValueError):
+ volume_db = 0.0
+
url = "https://api.fish.audio/v1/tts"
headers = {
"Authorization": f"Bearer {api_key}",
"Content-Type": "application/json",
- "model": model_name or "s2.1-pro-free",
+ "model": model_name,
}
- payload = {
+ payload: dict = {
"text": text,
"format": "mp3",
+ "prosody": {
+ "speed": speed,
+ "volume": volume_db,
+ },
}
+ if reference_id:
+ payload["reference_id"] = reference_id
for i in range(3):
try:
- logger.info(f"start fish audio tts, model: {model_name}, try: {i + 1}")
+ logger.info(
+ f"start fish audio tts, model: {model_name}, "
+ f"ref: {reference_id or 'default'}, try: {i + 1}"
+ )
ensure_file_path_exists(voice_file)
response = requests.post(url, json=payload, headers=headers, timeout=60)
+ if response.status_code == 401:
+ logger.error(
+ "Fish Audio TTS failed: Invalid API key (401). "
+ "Check config.toml [fish_audio] api_key or FISH_API_KEY."
+ )
+ return None
+ if response.status_code == 402:
+ logger.error(
+ "Fish Audio TTS failed: Insufficient API credit (402). "
+ "Please use model 's2.1-pro-free' or add funds at "
+ "https://fish.audio/app/developers"
+ )
+ return None
+ if response.status_code == 429:
+ logger.warning(
+ "Fish Audio TTS rate limited (429), retrying..."
+ )
+ continue
if response.status_code != 200:
- if response.status_code == 402:
- logger.error(
- "Fish Audio TTS failed: Insufficient API credit (402). "
- "Please use model 's2.1-pro-free' or visit https://fish.audio/app/developers to add funds."
- )
- return None
logger.error(
- f"fish audio tts failed with status {response.status_code}: {response.text[:200]}"
+ f"fish audio tts failed with status "
+ f"{response.status_code}: {response.text[:200]}"
+ )
+ continue
+
+ # Validate response contains audio data
+ if not response.content or len(response.content) < 100:
+ logger.error(
+ "Fish Audio TTS returned empty or invalid audio data"
)
continue
diff --git a/config.example.toml b/config.example.toml
index 3a5d170f..b19387fc 100644
--- a/config.example.toml
+++ b/config.example.toml
@@ -404,6 +404,22 @@ api_key = ""
# Available models: "s2.1-pro-free" (free developer tier), "s2.1-pro", "s2-pro"
model = "s2.1-pro-free"
+# ── Voice selection ──────────────────────────────────────────────────────────
+# By default Fish Audio uses its built-in default voice. To use a specific
+# public or cloned voice, add entries below. Each entry is either:
+# "<reference_id>:<display_name>" – a Fish Audio voice model ID with label
+# "<reference_id>" – bare ID (used as both ID and label)
+#
+# Browse public voices at https://fish.audio and copy the model ID from the URL.
+# Example: voices = ["7f92f8afb8ec43bf81429cc1c9199cb1:Friendly Narrator"]
+voices = []
+
+# ── Free tier notice ─────────────────────────────────────────────────────────
+# The "s2.1-pro-free" model is free for developers but is subject to Fish
+# Audio's Fair Use Policy. There is no SLA, and usage may be rate-limited.
+# Check https://fish.audio/pricing for current terms on commercial use and
+# data retention before using in production.
+
# =============================================================================
# WebUI Preferences / WebUI 偏好
# =============================================================================
diff --git a/test/services/test_fish_audio.py b/test/services/test_fish_audio.py
new file mode 100644
index 00000000..22e4d42b
--- /dev/null
+++ b/test/services/test_fish_audio.py
@@ -0,0 +1,284 @@
+"""Tests for Fish Audio TTS integration.
+
+Covers dispatch/model selection, config and environment API keys, request
+payloads (prosody, reference_id), 401/402/429 handling, invalid audio
+responses, voice helpers, and task restoration via _infer_tts_server_from_voice.
+"""
+
+import math
+import os
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+from unittest.mock import patch, MagicMock
+
+sys.path.insert(0, str(Path(__file__).parent.parent.parent))
+
+from app.services import voice as vs
+
+
+class _FakeClip:
+ duration = 4.2
+
+ def close(self):
+ pass
+
+
+class _FakeResponse:
+ def __init__(self, status_code=200, content=b"\xff" * 200, text=""):
+ self.status_code = status_code
+ self.content = content
+ self.text = text
+
+
+class TestFishAudioVoiceHelpers(unittest.TestCase):
+ """is_fish_audio_voice / get_fish_audio_voices basics."""
+
+ def test_is_fish_audio_voice_recognizes_prefix(self):
+ self.assertTrue(vs.is_fish_audio_voice("fish_audio:default:Default Voice"))
+ self.assertTrue(vs.is_fish_audio_voice("fish_audio:abc123:My Voice"))
+ self.assertFalse(vs.is_fish_audio_voice("elevenlabs:abc:Rachel"))
+ self.assertFalse(vs.is_fish_audio_voice(""))
+ self.assertFalse(vs.is_fish_audio_voice(None))
+
+ def test_get_fish_audio_voices_default(self):
+ """With no voices configured, returns the default voice entry."""
+ with patch.object(vs.config, "fish_audio", {"voices": []}):
+ voices = vs.get_fish_audio_voices()
+ self.assertEqual(voices, ["fish_audio:default:Default Voice"])
+
+ def test_get_fish_audio_voices_with_configured_entries(self):
+ """User-configured voices are appended after the default."""
+ with patch.object(
+ vs.config,
+ "fish_audio",
+ {"voices": ["abc123:My Narrator", "def456"]},
+ ):
+ voices = vs.get_fish_audio_voices()
+ self.assertEqual(
+ voices,
+ [
+ "fish_audio:default:Default Voice",
+ "fish_audio:abc123:My Narrator",
+ "fish_audio:def456:def456",
+ ],
+ )
+
+ def test_get_fish_audio_voices_comma_separated_string(self):
+ """TOML-friendly comma-separated string is accepted."""
+ with patch.object(
+ vs.config, "fish_audio", {"voices": "abc123:Voice A, def456:Voice B,"}
+ ):
+ voices = vs.get_fish_audio_voices()
+ self.assertIn("fish_audio:abc123:Voice A", voices)
+ self.assertIn("fish_audio:def456:Voice B", voices)
+
+ def test_get_fish_audio_voices_already_prefixed(self):
+ """Entries already prefixed with fish_audio: are kept as-is."""
+ with patch.object(
+ vs.config,
+ "fish_audio",
+ {"voices": ["fish_audio:abc:Custom"]},
+ ):
+ voices = vs.get_fish_audio_voices()
+ self.assertIn("fish_audio:abc:Custom", voices)
+
+
+class TestFishAudioAPIKey(unittest.TestCase):
+ """get_fish_audio_api_key reads from config and env."""
+
+ def test_api_key_from_config(self):
+ with patch.object(
+ vs.config, "fish_audio", {"api_key": "config-key-123"}
+ ):
+ self.assertEqual(vs.get_fish_audio_api_key(), "config-key-123")
+
+ def test_api_key_from_env_fallback(self):
+ with patch.object(vs.config, "fish_audio", {"api_key": ""}), \
+ patch.dict(os.environ, {"FISH_API_KEY": "env-key-456"}):
+ self.assertEqual(vs.get_fish_audio_api_key(), "env-key-456")
+
+ def test_api_key_config_takes_precedence(self):
+ with patch.object(
+ vs.config, "fish_audio", {"api_key": "config-key"}
+ ), patch.dict(os.environ, {"FISH_API_KEY": "env-key"}):
+ self.assertEqual(vs.get_fish_audio_api_key(), "config-key")
+
+
+class TestFishAudioDispatch(unittest.TestCase):
+ """tts() correctly dispatches to fish_audio_tts."""
+
+ def test_dispatch_default_voice(self):
+ sentinel = object()
+ with patch.object(vs, "fish_audio_tts", return_value=sentinel) as mock:
+ result = vs.tts("hello", "fish_audio:default:Default Voice", 1.0, "out.mp3", 1.0)
+ self.assertIs(result, sentinel)
+ mock.assert_called_once_with(
+ "hello", "out.mp3", 1.0, 1.0, reference_id=None
+ )
+
+ def test_dispatch_custom_reference_id(self):
+ sentinel = object()
+ with patch.object(vs, "fish_audio_tts", return_value=sentinel) as mock:
+ result = vs.tts("hello", "fish_audio:abc123:My Voice", 1.2, "out.mp3", 0.8)
+ self.assertIs(result, sentinel)
+ mock.assert_called_once_with(
+ "hello", "out.mp3", 1.2, 0.8, reference_id="abc123"
+ )
+
+
+class TestFishAudioTTSRequest(unittest.TestCase):
+ """fish_audio_tts sends correct request payloads."""
+
+ def _call_with_capture(self, voice_rate=1.0, voice_volume=1.0,
+ reference_id=None, model="s2.1-pro-free",
+ response=None):
+ """Helper: call fish_audio_tts and capture the outgoing request."""
+ captured = {}
+
+ def _fake_post(url, json=None, headers=None, timeout=None):
+ captured["url"] = url
+ captured["json"] = json
+ captured["headers"] = headers
+ return response or _FakeResponse()
+
+ with tempfile.TemporaryDirectory() as tmp_dir, \
+ patch.object(vs.config, "fish_audio", {"api_key": "test-key", "model": model}), \
+ patch.object(vs.requests, "post", side_effect=_fake_post), \
+ patch.object(vs, "AudioFileClip", return_value=_FakeClip()):
+ voice_file = str(Path(tmp_dir) / "fish.mp3")
+ result = vs.fish_audio_tts(
+ "Test sentence.",
+ voice_file,
+ voice_rate=voice_rate,
+ voice_volume=voice_volume,
+ reference_id=reference_id,
+ )
+ return result, captured
+
+ def test_model_from_config(self):
+ """Model is read from config, not from voice name."""
+ _, captured = self._call_with_capture(model="s2.1-pro")
+ self.assertEqual(captured["headers"]["model"], "s2.1-pro")
+
+ def test_default_model_fallback(self):
+ """Unknown model falls back to s2.1-pro-free."""
+ _, captured = self._call_with_capture(model="nonexistent-model")
+ self.assertEqual(captured["headers"]["model"], "s2.1-pro-free")
+
+ def test_prosody_speed_sent(self):
+ """voice_rate is mapped to prosody.speed."""
+ _, captured = self._call_with_capture(voice_rate=1.5)
+ self.assertAlmostEqual(captured["json"]["prosody"]["speed"], 1.5)
+
+ def test_prosody_speed_clamped(self):
+ """voice_rate is clamped to 0.5–2.0."""
+ _, cap_low = self._call_with_capture(voice_rate=0.1)
+ self.assertAlmostEqual(cap_low["json"]["prosody"]["speed"], 0.5)
+ _, cap_high = self._call_with_capture(voice_rate=5.0)
+ self.assertAlmostEqual(cap_high["json"]["prosody"]["speed"], 2.0)
+
+ def test_prosody_volume_conversion(self):
+ """voice_volume linear multiplier is converted to dB."""
+ # 1.0 → 0 dB
+ _, cap = self._call_with_capture(voice_volume=1.0)
+ self.assertAlmostEqual(cap["json"]["prosody"]["volume"], 0.0, places=1)
+ # 2.0 → ~6 dB
+ _, cap2 = self._call_with_capture(voice_volume=2.0)
+ self.assertAlmostEqual(
+ cap2["json"]["prosody"]["volume"],
+ 20.0 * math.log10(2.0),
+ places=1,
+ )
+
+ def test_reference_id_included_when_set(self):
+ _, captured = self._call_with_capture(reference_id="abc123")
+ self.assertEqual(captured["json"]["reference_id"], "abc123")
+
+ def test_reference_id_absent_when_none(self):
+ _, captured = self._call_with_capture(reference_id=None)
+ self.assertNotIn("reference_id", captured["json"])
+
+ def test_auth_header(self):
+ _, captured = self._call_with_capture()
+ self.assertEqual(captured["headers"]["Authorization"], "Bearer test-key")
+
+ def test_success_returns_submaker(self):
+ result, _ = self._call_with_capture()
+ self.assertIsNotNone(result)
+
+
+class TestFishAudioErrorHandling(unittest.TestCase):
+ """Error responses are handled gracefully."""
+
+ def _call_with_status(self, status_code, text="error"):
+ resp = _FakeResponse(status_code=status_code, content=b"", text=text)
+
+ with tempfile.TemporaryDirectory() as tmp_dir, \
+ patch.object(vs.config, "fish_audio", {"api_key": "test-key", "model": "s2.1-pro-free"}), \
+ patch.object(vs.requests, "post", return_value=resp), \
+ patch.object(vs, "AudioFileClip", return_value=_FakeClip()):
+ voice_file = str(Path(tmp_dir) / "fish.mp3")
+ result = vs.fish_audio_tts("Test.", voice_file)
+ return result
+
+ def test_401_returns_none(self):
+ self.assertIsNone(self._call_with_status(401))
+
+ def test_402_returns_none(self):
+ self.assertIsNone(self._call_with_status(402))
+
+ def test_429_returns_none_after_retries(self):
+ """429 triggers retry; after 3 attempts returns None."""
+ self.assertIsNone(self._call_with_status(429))
+
+ def test_500_returns_none_after_retries(self):
+ self.assertIsNone(self._call_with_status(500))
+
+ def test_empty_audio_returns_none(self):
+ """Response with too-small content is rejected."""
+ resp = _FakeResponse(status_code=200, content=b"tiny")
+ with tempfile.TemporaryDirectory() as tmp_dir, \
+ patch.object(vs.config, "fish_audio", {"api_key": "test-key", "model": "s2.1-pro-free"}), \
+ patch.object(vs.requests, "post", return_value=resp):
+ voice_file = str(Path(tmp_dir) / "fish.mp3")
+ result = vs.fish_audio_tts("Test.", voice_file)
+ self.assertIsNone(result)
+
+ def test_missing_api_key_returns_none(self):
+ with patch.object(vs.config, "fish_audio", {"api_key": ""}), \
+ patch.dict(os.environ, {"FISH_API_KEY": ""}, clear=False):
+ result = vs.fish_audio_tts("Test.", "/tmp/fish.mp3")
+ self.assertIsNone(result)
+
+ def test_empty_text_returns_none(self):
+ result = vs.fish_audio_tts("", "/tmp/fish.mp3")
+ self.assertIsNone(result)
+
+
+class TestFishAudioTaskRestore(unittest.TestCase):
+ """Verify that fish_audio voices are detectable for task restoration.
+
+ _infer_tts_server_from_voice (in webui/Main.py) delegates to
+ voice.is_fish_audio_voice(), so we test that the detection function
+ correctly identifies fish_audio voice strings — which is the root
+ cause of the task restore bug the maintainer reported.
+ """
+
+ def test_is_fish_audio_voice_matches_default(self):
+ self.assertTrue(vs.is_fish_audio_voice("fish_audio:default:Default Voice"))
+
+ def test_is_fish_audio_voice_matches_custom_ref(self):
+ self.assertTrue(vs.is_fish_audio_voice("fish_audio:abc123:Custom"))
+
+ def test_is_fish_audio_voice_rejects_other_providers(self):
+ self.assertFalse(vs.is_fish_audio_voice("elevenlabs:abc:Rachel"))
+ self.assertFalse(vs.is_fish_audio_voice("chatterbox:default-Female"))
+ self.assertFalse(vs.is_fish_audio_voice("minimax:narrator"))
+ # Azure voices don't have a prefix
+ self.assertFalse(vs.is_fish_audio_voice("en-US-JennyNeural-Female"))
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/webui.sh b/webui.sh
index 866df9cf..27027c45 100644
--- a/webui.sh
+++ b/webui.sh
@@ -17,14 +17,14 @@ MPT_WEBUI_HOST="${MPT_WEBUI_HOST:-127.0.0.1}"
MPT_WEBUI_PORT="${MPT_WEBUI_PORT:-8501}"
if [ -x "$CURRENT_DIR/.venv/bin/python" ]; then
- PORT_CHECK_EXEC="$CURRENT_DIR/.venv/bin/python"
+ PORT_CHECK_CMD="$CURRENT_DIR/.venv/bin/python"
set -- "$CURRENT_DIR/.venv/bin/python" -m streamlit
elif command -v uv >/dev/null 2>&1; then
- PORT_CHECK_EXEC="uv"
+ PORT_CHECK_CMD="uv run python"
set -- uv run streamlit
elif command -v streamlit >/dev/null 2>&1; then
echo "***** Warning: using streamlit from PATH. If dependencies fail, run 'uv sync --frozen' first. *****"
- PORT_CHECK_EXEC="python3"
+ PORT_CHECK_CMD="python3"
set -- streamlit
else
echo "***** Neither project Python, uv, nor streamlit was found. Please install dependencies first. *****"
@@ -32,8 +32,7 @@ else
fi
find_available_port() {
- if [ "$1" = "uv" ]; then
- WEBUI_HOST="$MPT_WEBUI_HOST" WEBUI_PORT="$MPT_WEBUI_PORT" uv run python - <<'PY' 2>/dev/null
+ WEBUI_HOST="$MPT_WEBUI_HOST" WEBUI_PORT="$MPT_WEBUI_PORT" "$@" - <<'PY' 2>/dev/null
import os
import socket
import sys
@@ -53,32 +52,11 @@ for port in candidates:
sys.exit(1)
PY
- else
- WEBUI_HOST="$MPT_WEBUI_HOST" WEBUI_PORT="$MPT_WEBUI_PORT" "$1" - <<'PY' 2>/dev/null
-import os
-import socket
-import sys
-
-host = os.environ.get("WEBUI_HOST", "127.0.0.1")
-preferred = int(os.environ.get("WEBUI_PORT", "8501"))
-candidates = [preferred] + [port for port in range(8502, 8600) if port != preferred]
-
-for port in candidates:
- with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
- try:
- sock.bind((host, port))
- except OSError:
- continue
- print(port)
- sys.exit(0)
-
-sys.exit(1)
-PY
- fi
}
# 用 Python 做端口探测,避免依赖 lsof/nc 在不同 macOS/Linux 发行版上的差异。
-SELECTED_WEBUI_PORT=$(find_available_port "$PORT_CHECK_EXEC")
+# shellcheck disable=SC2086
+SELECTED_WEBUI_PORT=$(find_available_port $PORT_CHECK_CMD)
if [ -z "$SELECTED_WEBUI_PORT" ]; then
echo "***** No available WebUI port found in 8501-8599 for $MPT_WEBUI_HOST. *****"
diff --git a/webui/Main.py b/webui/Main.py
index a83630ea..9f4aaaf5 100644
--- a/webui/Main.py
+++ b/webui/Main.py
@@ -1165,6 +1165,8 @@ def _infer_tts_server_from_voice(voice_name):
return "elevenlabs"
if voice.is_chatterbox_voice(voice_name):
return "chatterbox"
+ if voice.is_fish_audio_voice(voice_name):
+ return "fish_audio"
if voice.is_azure_v2_voice(voice_name):
return "azure-tts-v2"
return "azure-tts-v1"
@@ -4873,7 +4875,8 @@ def _render_audio_settings(panel, params):
if voice.is_minimax_voice(v):
return minimax_voice_labels.get(v, v.split(":", 1)[1])
if voice.is_fish_audio_voice(v):
- return v
+ parts = v.split(":", 2)
+ return parts[2] if len(parts) >= 3 else v
return (
v.replace("Female", tr("Female"))
.replace("Male", tr("Male"))