利用 AI 大模型和自动化工作流,根据主题或关键词一键生成高清短视频。Generate HD short videos from a topic or keyword with an automated AI workflow.
Python ai-video-generatorcontent-creationffmpeginstagram-reelsllmpythonshort-videosubtitlestext-to-speechtiktokvideo-automationvideo-workflowworkflow-automationyoutube-shorts
fix(tts): address all maintainer review feedback on Fish Audio integration
Hariom paswan375a45806f2f
diff --git a/app/services/voice.py b/app/services/voice.pyindex d4d46f2a..7266fa14 100644--- a/app/services/voice.py+++ b/app/services/voice.py@@ -222,12 +222,31 @@ def get_chatterbox_voices() -> list[str]:def get_fish_audio_voices() -> list[str]:- """获取 Fish Audio 预置声音与模型列表。"""- return [- "fish_audio:s2.1-pro-free:Free-Developer",- "fish_audio:s2.1-pro:Pro-Voice",- "fish_audio:s2-pro:Pro-Voice",- ]+ """Return configured Fish Audio voices.++ Each entry follows the format ``fish_audio:<reference_id>:<display_name>``.+ When ``reference_id`` is "default", Fish Audio's built-in default voice is+ used (no ``reference_id`` is sent in the API request). Operators can list+ additional public or cloned voices via ``[fish_audio] voices`` in the+ config file.+ """+ result = ["fish_audio:default:Default Voice"]+ voices = config.fish_audio.get("voices", []) or []+ if isinstance(voices, str):+ voices = [v.strip() for v in voices.split(",") if v.strip()]+ for entry in voices:+ entry = str(entry).strip()+ if not entry:+ continue+ if entry.startswith("fish_audio:"):+ result.append(entry)+ elif ":" in entry:+ # "<reference_id>:<display_name>"+ result.append(f"fish_audio:{entry}")+ else:+ # bare reference_id+ result.append(f"fish_audio:{entry}:{entry}")+ return result_AZURE_VOICES_DATA_FILE = os.path.join(@@ -523,8 +542,10 @@ def tts(return Noneelif is_fish_audio_voice(voice_name):parts = voice_name.split(":")- model_name = parts[1] if len(parts) >= 2 else "s2.1-pro-free"- return fish_audio_tts(text, model_name, voice_file, voice_rate, voice_volume)+ reference_id = parts[1] if len(parts) >= 2 else "default"+ if reference_id == "default":+ reference_id = None+ return fish_audio_tts(text, voice_file, voice_rate, voice_volume, reference_id=reference_id)return azure_tts_v1(text, voice_name, voice_rate, voice_file)@@ -1764,14 +1785,28 @@ def chatterbox_tts(return None+# Fish Audio supported models.+FISH_AUDIO_MODELS = ("s2.1-pro-free", "s2.1-pro", "s2-pro")+FISH_AUDIO_DEFAULT_MODEL = "s2.1-pro-free"++def fish_audio_tts(text: str,- model_name: str,voice_file: str,voice_rate: float = 1.0,voice_volume: float = 1.0,+ reference_id: str | None = None,) -> Union[SubMaker, None]:- """Generate speech using Fish Audio TTS API."""+ """Generate speech using Fish Audio TTS API.++ The model is read from ``config.fish_audio["model"]`` (single source of+ truth). ``reference_id`` selects a public or cloned voice; when *None*+ Fish Audio's built-in default voice is used.++ ``voice_rate`` is mapped to the ``prosody.speed`` field (0.5–2.0) and+ ``voice_volume`` is converted from a linear multiplier to dB for the+ ``prosody.volume`` field (-20.0–20.0 dB).+ """text = (text or "").strip()if not text:logger.error("Fish Audio TTS text is empty")@@ -1780,36 +1815,95 @@ def fish_audio_tts(api_key = get_fish_audio_api_key()if not api_key:logger.error(- "Fish Audio API key is not set. Please set it in config.toml [fish_audio] or FISH_API_KEY environment variable."+ "Fish Audio API key is not set. Please set it in config.toml "+ "[fish_audio] or FISH_API_KEY environment variable.")return None+ model_name = str(+ config.fish_audio.get("model", FISH_AUDIO_DEFAULT_MODEL)+ or FISH_AUDIO_DEFAULT_MODEL+ ).strip()+ if model_name not in FISH_AUDIO_MODELS:+ logger.warning(+ f"Unknown Fish Audio model '{model_name}', falling back to "+ f"'{FISH_AUDIO_DEFAULT_MODEL}'"+ )+ model_name = FISH_AUDIO_DEFAULT_MODEL++ # Map voice_rate → prosody.speed (0.5–2.0)+ try:+ speed = max(0.5, min(2.0, float(voice_rate or 1.0)))+ except (TypeError, ValueError):+ speed = 1.0++ # Map voice_volume (linear multiplier) → prosody.volume (dB, -20–20).+ # A multiplier of 1.0 → 0 dB; 0.1 → -20 dB; 2.0 → +6 dB.+ import math+ try:+ vol = float(voice_volume or 1.0)+ if vol <= 0:+ volume_db = -20.0+ else:+ volume_db = max(-20.0, min(20.0, 20.0 * math.log10(vol)))+ except (TypeError, ValueError):+ volume_db = 0.0+url = "https://api.fish.audio/v1/tts"headers = {"Authorization": f"Bearer {api_key}","Content-Type": "application/json",- "model": model_name or "s2.1-pro-free",+ "model": model_name,}- payload = {+ payload: dict = {"text": text,"format": "mp3",+ "prosody": {+ "speed": speed,+ "volume": volume_db,+ },}+ if reference_id:+ payload["reference_id"] = reference_idfor i in range(3):try:- logger.info(f"start fish audio tts, model: {model_name}, try: {i + 1}")+ logger.info(+ f"start fish audio tts, model: {model_name}, "+ f"ref: {reference_id or 'default'}, try: {i + 1}"+ )ensure_file_path_exists(voice_file)response = requests.post(url, json=payload, headers=headers, timeout=60)+ if response.status_code == 401:+ logger.error(+ "Fish Audio TTS failed: Invalid API key (401). "+ "Check config.toml [fish_audio] api_key or FISH_API_KEY."+ )+ return None+ if response.status_code == 402:+ logger.error(+ "Fish Audio TTS failed: Insufficient API credit (402). "+ "Please use model 's2.1-pro-free' or add funds at "+ "https://fish.audio/app/developers"+ )+ return None+ if response.status_code == 429:+ logger.warning(+ "Fish Audio TTS rate limited (429), retrying..."+ )+ continueif response.status_code != 200:- if response.status_code == 402:- logger.error(- "Fish Audio TTS failed: Insufficient API credit (402). "- "Please use model 's2.1-pro-free' or visit https://fish.audio/app/developers to add funds."- )- return Nonelogger.error(- f"fish audio tts failed with status {response.status_code}: {response.text[:200]}"+ f"fish audio tts failed with status "+ f"{response.status_code}: {response.text[:200]}"+ )+ continue++ # Validate response contains audio data+ if not response.content or len(response.content) < 100:+ logger.error(+ "Fish Audio TTS returned empty or invalid audio data")continuediff --git a/config.example.toml b/config.example.tomlindex 3a5d170f..b19387fc 100644--- a/config.example.toml+++ b/config.example.toml@@ -404,6 +404,22 @@ api_key = ""# Available models: "s2.1-pro-free" (free developer tier), "s2.1-pro", "s2-pro"model = "s2.1-pro-free"+# ── Voice selection ──────────────────────────────────────────────────────────+# By default Fish Audio uses its built-in default voice. To use a specific+# public or cloned voice, add entries below. Each entry is either:+# "<reference_id>:<display_name>" – a Fish Audio voice model ID with label+# "<reference_id>" – bare ID (used as both ID and label)+#+# Browse public voices at https://fish.audio and copy the model ID from the URL.+# Example: voices = ["7f92f8afb8ec43bf81429cc1c9199cb1:Friendly Narrator"]+voices = []++# ── Free tier notice ─────────────────────────────────────────────────────────+# The "s2.1-pro-free" model is free for developers but is subject to Fish+# Audio's Fair Use Policy. There is no SLA, and usage may be rate-limited.+# Check https://fish.audio/pricing for current terms on commercial use and+# data retention before using in production.+# =============================================================================# WebUI Preferences / WebUI 偏好# =============================================================================diff --git a/test/services/test_fish_audio.py b/test/services/test_fish_audio.pynew file mode 100644index 00000000..22e4d42b--- /dev/null+++ b/test/services/test_fish_audio.py@@ -0,0 +1,284 @@+"""Tests for Fish Audio TTS integration.++Covers dispatch/model selection, config and environment API keys, request+payloads (prosody, reference_id), 401/402/429 handling, invalid audio+responses, voice helpers, and task restoration via _infer_tts_server_from_voice.+"""++import math+import os+import sys+import tempfile+import unittest+from pathlib import Path+from unittest.mock import patch, MagicMock++sys.path.insert(0, str(Path(__file__).parent.parent.parent))++from app.services import voice as vs+++class _FakeClip:+ duration = 4.2++ def close(self):+ pass+++class _FakeResponse:+ def __init__(self, status_code=200, content=b"\xff" * 200, text=""):+ self.status_code = status_code+ self.content = content+ self.text = text+++class TestFishAudioVoiceHelpers(unittest.TestCase):+ """is_fish_audio_voice / get_fish_audio_voices basics."""++ def test_is_fish_audio_voice_recognizes_prefix(self):+ self.assertTrue(vs.is_fish_audio_voice("fish_audio:default:Default Voice"))+ self.assertTrue(vs.is_fish_audio_voice("fish_audio:abc123:My Voice"))+ self.assertFalse(vs.is_fish_audio_voice("elevenlabs:abc:Rachel"))+ self.assertFalse(vs.is_fish_audio_voice(""))+ self.assertFalse(vs.is_fish_audio_voice(None))++ def test_get_fish_audio_voices_default(self):+ """With no voices configured, returns the default voice entry."""+ with patch.object(vs.config, "fish_audio", {"voices": []}):+ voices = vs.get_fish_audio_voices()+ self.assertEqual(voices, ["fish_audio:default:Default Voice"])++ def test_get_fish_audio_voices_with_configured_entries(self):+ """User-configured voices are appended after the default."""+ with patch.object(+ vs.config,+ "fish_audio",+ {"voices": ["abc123:My Narrator", "def456"]},+ ):+ voices = vs.get_fish_audio_voices()+ self.assertEqual(+ voices,+ [+ "fish_audio:default:Default Voice",+ "fish_audio:abc123:My Narrator",+ "fish_audio:def456:def456",+ ],+ )++ def test_get_fish_audio_voices_comma_separated_string(self):+ """TOML-friendly comma-separated string is accepted."""+ with patch.object(+ vs.config, "fish_audio", {"voices": "abc123:Voice A, def456:Voice B,"}+ ):+ voices = vs.get_fish_audio_voices()+ self.assertIn("fish_audio:abc123:Voice A", voices)+ self.assertIn("fish_audio:def456:Voice B", voices)++ def test_get_fish_audio_voices_already_prefixed(self):+ """Entries already prefixed with fish_audio: are kept as-is."""+ with patch.object(+ vs.config,+ "fish_audio",+ {"voices": ["fish_audio:abc:Custom"]},+ ):+ voices = vs.get_fish_audio_voices()+ self.assertIn("fish_audio:abc:Custom", voices)+++class TestFishAudioAPIKey(unittest.TestCase):+ """get_fish_audio_api_key reads from config and env."""++ def test_api_key_from_config(self):+ with patch.object(+ vs.config, "fish_audio", {"api_key": "config-key-123"}+ ):+ self.assertEqual(vs.get_fish_audio_api_key(), "config-key-123")++ def test_api_key_from_env_fallback(self):+ with patch.object(vs.config, "fish_audio", {"api_key": ""}), \+ patch.dict(os.environ, {"FISH_API_KEY": "env-key-456"}):+ self.assertEqual(vs.get_fish_audio_api_key(), "env-key-456")++ def test_api_key_config_takes_precedence(self):+ with patch.object(+ vs.config, "fish_audio", {"api_key": "config-key"}+ ), patch.dict(os.environ, {"FISH_API_KEY": "env-key"}):+ self.assertEqual(vs.get_fish_audio_api_key(), "config-key")+++class TestFishAudioDispatch(unittest.TestCase):+ """tts() correctly dispatches to fish_audio_tts."""++ def test_dispatch_default_voice(self):+ sentinel = object()+ with patch.object(vs, "fish_audio_tts", return_value=sentinel) as mock:+ result = vs.tts("hello", "fish_audio:default:Default Voice", 1.0, "out.mp3", 1.0)+ self.assertIs(result, sentinel)+ mock.assert_called_once_with(+ "hello", "out.mp3", 1.0, 1.0, reference_id=None+ )++ def test_dispatch_custom_reference_id(self):+ sentinel = object()+ with patch.object(vs, "fish_audio_tts", return_value=sentinel) as mock:+ result = vs.tts("hello", "fish_audio:abc123:My Voice", 1.2, "out.mp3", 0.8)+ self.assertIs(result, sentinel)+ mock.assert_called_once_with(+ "hello", "out.mp3", 1.2, 0.8, reference_id="abc123"+ )+++class TestFishAudioTTSRequest(unittest.TestCase):+ """fish_audio_tts sends correct request payloads."""++ def _call_with_capture(self, voice_rate=1.0, voice_volume=1.0,+ reference_id=None, model="s2.1-pro-free",+ response=None):+ """Helper: call fish_audio_tts and capture the outgoing request."""+ captured = {}++ def _fake_post(url, json=None, headers=None, timeout=None):+ captured["url"] = url+ captured["json"] = json+ captured["headers"] = headers+ return response or _FakeResponse()++ with tempfile.TemporaryDirectory() as tmp_dir, \+ patch.object(vs.config, "fish_audio", {"api_key": "test-key", "model": model}), \+ patch.object(vs.requests, "post", side_effect=_fake_post), \+ patch.object(vs, "AudioFileClip", return_value=_FakeClip()):+ voice_file = str(Path(tmp_dir) / "fish.mp3")+ result = vs.fish_audio_tts(+ "Test sentence.",+ voice_file,+ voice_rate=voice_rate,+ voice_volume=voice_volume,+ reference_id=reference_id,+ )+ return result, captured++ def test_model_from_config(self):+ """Model is read from config, not from voice name."""+ _, captured = self._call_with_capture(model="s2.1-pro")+ self.assertEqual(captured["headers"]["model"], "s2.1-pro")++ def test_default_model_fallback(self):+ """Unknown model falls back to s2.1-pro-free."""+ _, captured = self._call_with_capture(model="nonexistent-model")+ self.assertEqual(captured["headers"]["model"], "s2.1-pro-free")++ def test_prosody_speed_sent(self):+ """voice_rate is mapped to prosody.speed."""+ _, captured = self._call_with_capture(voice_rate=1.5)+ self.assertAlmostEqual(captured["json"]["prosody"]["speed"], 1.5)++ def test_prosody_speed_clamped(self):+ """voice_rate is clamped to 0.5–2.0."""+ _, cap_low = self._call_with_capture(voice_rate=0.1)+ self.assertAlmostEqual(cap_low["json"]["prosody"]["speed"], 0.5)+ _, cap_high = self._call_with_capture(voice_rate=5.0)+ self.assertAlmostEqual(cap_high["json"]["prosody"]["speed"], 2.0)++ def test_prosody_volume_conversion(self):+ """voice_volume linear multiplier is converted to dB."""+ # 1.0 → 0 dB+ _, cap = self._call_with_capture(voice_volume=1.0)+ self.assertAlmostEqual(cap["json"]["prosody"]["volume"], 0.0, places=1)+ # 2.0 → ~6 dB+ _, cap2 = self._call_with_capture(voice_volume=2.0)+ self.assertAlmostEqual(+ cap2["json"]["prosody"]["volume"],+ 20.0 * math.log10(2.0),+ places=1,+ )++ def test_reference_id_included_when_set(self):+ _, captured = self._call_with_capture(reference_id="abc123")+ self.assertEqual(captured["json"]["reference_id"], "abc123")++ def test_reference_id_absent_when_none(self):+ _, captured = self._call_with_capture(reference_id=None)+ self.assertNotIn("reference_id", captured["json"])++ def test_auth_header(self):+ _, captured = self._call_with_capture()+ self.assertEqual(captured["headers"]["Authorization"], "Bearer test-key")++ def test_success_returns_submaker(self):+ result, _ = self._call_with_capture()+ self.assertIsNotNone(result)+++class TestFishAudioErrorHandling(unittest.TestCase):+ """Error responses are handled gracefully."""++ def _call_with_status(self, status_code, text="error"):+ resp = _FakeResponse(status_code=status_code, content=b"", text=text)++ with tempfile.TemporaryDirectory() as tmp_dir, \+ patch.object(vs.config, "fish_audio", {"api_key": "test-key", "model": "s2.1-pro-free"}), \+ patch.object(vs.requests, "post", return_value=resp), \+ patch.object(vs, "AudioFileClip", return_value=_FakeClip()):+ voice_file = str(Path(tmp_dir) / "fish.mp3")+ result = vs.fish_audio_tts("Test.", voice_file)+ return result++ def test_401_returns_none(self):+ self.assertIsNone(self._call_with_status(401))++ def test_402_returns_none(self):+ self.assertIsNone(self._call_with_status(402))++ def test_429_returns_none_after_retries(self):+ """429 triggers retry; after 3 attempts returns None."""+ self.assertIsNone(self._call_with_status(429))++ def test_500_returns_none_after_retries(self):+ self.assertIsNone(self._call_with_status(500))++ def test_empty_audio_returns_none(self):+ """Response with too-small content is rejected."""+ resp = _FakeResponse(status_code=200, content=b"tiny")+ with tempfile.TemporaryDirectory() as tmp_dir, \+ patch.object(vs.config, "fish_audio", {"api_key": "test-key", "model": "s2.1-pro-free"}), \+ patch.object(vs.requests, "post", return_value=resp):+ voice_file = str(Path(tmp_dir) / "fish.mp3")+ result = vs.fish_audio_tts("Test.", voice_file)+ self.assertIsNone(result)++ def test_missing_api_key_returns_none(self):+ with patch.object(vs.config, "fish_audio", {"api_key": ""}), \+ patch.dict(os.environ, {"FISH_API_KEY": ""}, clear=False):+ result = vs.fish_audio_tts("Test.", "/tmp/fish.mp3")+ self.assertIsNone(result)++ def test_empty_text_returns_none(self):+ result = vs.fish_audio_tts("", "/tmp/fish.mp3")+ self.assertIsNone(result)+++class TestFishAudioTaskRestore(unittest.TestCase):+ """Verify that fish_audio voices are detectable for task restoration.++ _infer_tts_server_from_voice (in webui/Main.py) delegates to+ voice.is_fish_audio_voice(), so we test that the detection function+ correctly identifies fish_audio voice strings — which is the root+ cause of the task restore bug the maintainer reported.+ """++ def test_is_fish_audio_voice_matches_default(self):+ self.assertTrue(vs.is_fish_audio_voice("fish_audio:default:Default Voice"))++ def test_is_fish_audio_voice_matches_custom_ref(self):+ self.assertTrue(vs.is_fish_audio_voice("fish_audio:abc123:Custom"))++ def test_is_fish_audio_voice_rejects_other_providers(self):+ self.assertFalse(vs.is_fish_audio_voice("elevenlabs:abc:Rachel"))+ self.assertFalse(vs.is_fish_audio_voice("chatterbox:default-Female"))+ self.assertFalse(vs.is_fish_audio_voice("minimax:narrator"))+ # Azure voices don't have a prefix+ self.assertFalse(vs.is_fish_audio_voice("en-US-JennyNeural-Female"))+++if __name__ == "__main__":+ unittest.main()diff --git a/webui.sh b/webui.shindex 866df9cf..27027c45 100644--- a/webui.sh+++ b/webui.sh@@ -17,14 +17,14 @@ MPT_WEBUI_HOST="${MPT_WEBUI_HOST:-127.0.0.1}"MPT_WEBUI_PORT="${MPT_WEBUI_PORT:-8501}"if [ -x "$CURRENT_DIR/.venv/bin/python" ]; then- PORT_CHECK_EXEC="$CURRENT_DIR/.venv/bin/python"+ PORT_CHECK_CMD="$CURRENT_DIR/.venv/bin/python"set -- "$CURRENT_DIR/.venv/bin/python" -m streamlitelif command -v uv >/dev/null 2>&1; then- PORT_CHECK_EXEC="uv"+ PORT_CHECK_CMD="uv run python"set -- uv run streamlitelif command -v streamlit >/dev/null 2>&1; thenecho "***** Warning: using streamlit from PATH. If dependencies fail, run 'uv sync --frozen' first. *****"- PORT_CHECK_EXEC="python3"+ PORT_CHECK_CMD="python3"set -- streamlitelseecho "***** Neither project Python, uv, nor streamlit was found. Please install dependencies first. *****"@@ -32,8 +32,7 @@ elsefifind_available_port() {- if [ "$1" = "uv" ]; then- WEBUI_HOST="$MPT_WEBUI_HOST" WEBUI_PORT="$MPT_WEBUI_PORT" uv run python - <<'PY' 2>/dev/null+ WEBUI_HOST="$MPT_WEBUI_HOST" WEBUI_PORT="$MPT_WEBUI_PORT" "$@" - <<'PY' 2>/dev/nullimport osimport socketimport sys@@ -53,32 +52,11 @@ for port in candidates:sys.exit(1)PY- else- WEBUI_HOST="$MPT_WEBUI_HOST" WEBUI_PORT="$MPT_WEBUI_PORT" "$1" - <<'PY' 2>/dev/null-import os-import socket-import sys--host = os.environ.get("WEBUI_HOST", "127.0.0.1")-preferred = int(os.environ.get("WEBUI_PORT", "8501"))-candidates = [preferred] + [port for port in range(8502, 8600) if port != preferred]--for port in candidates:- with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:- try:- sock.bind((host, port))- except OSError:- continue- print(port)- sys.exit(0)--sys.exit(1)-PY- fi}# 用 Python 做端口探测,避免依赖 lsof/nc 在不同 macOS/Linux 发行版上的差异。-SELECTED_WEBUI_PORT=$(find_available_port "$PORT_CHECK_EXEC")+# shellcheck disable=SC2086+SELECTED_WEBUI_PORT=$(find_available_port $PORT_CHECK_CMD)if [ -z "$SELECTED_WEBUI_PORT" ]; thenecho "***** No available WebUI port found in 8501-8599 for $MPT_WEBUI_HOST. *****"diff --git a/webui/Main.py b/webui/Main.pyindex a83630ea..9f4aaaf5 100644--- a/webui/Main.py+++ b/webui/Main.py@@ -1165,6 +1165,8 @@ def _infer_tts_server_from_voice(voice_name):return "elevenlabs"if voice.is_chatterbox_voice(voice_name):return "chatterbox"+ if voice.is_fish_audio_voice(voice_name):+ return "fish_audio"if voice.is_azure_v2_voice(voice_name):return "azure-tts-v2"return "azure-tts-v1"@@ -4873,7 +4875,8 @@ def _render_audio_settings(panel, params):if voice.is_minimax_voice(v):return minimax_voice_labels.get(v, v.split(":", 1)[1])if voice.is_fish_audio_voice(v):- return v+ parts = v.split(":", 2)+ return parts[2] if len(parts) >= 3 else vreturn (v.replace("Female", tr("Female")).replace("Male", tr("Male"))