Skip to content

利用 AI 大模型和自动化工作流,根据主题或关键词一键生成高清短视频。Generate HD short videos from a topic or keyword with an automated AI workflow.

refactor: enhance pause tag validation, filtering logic, and multi-segment audio synchronization accuracy

Ismael Camachodc67d6cfc7d8
diff --git a/app/services/voice.py b/app/services/voice.py
index 54cebb26..632e941f 100644
--- a/app/services/voice.py
+++ b/app/services/voice.py
@@ -13,6 +13,7 @@ import tempfile
import threading
import time
import unicodedata
+import wave
from datetime import datetime, timedelta
from typing import Union
from urllib.parse import urlparse
@@ -427,6 +428,39 @@ def is_no_voice(voice_name: str | None) -> bool:
return str(voice_name or "").strip().lower() in _NO_VOICE_ALIASES
+def is_azure_v1_voice(voice_name: str | None) -> bool:
+ """
+ 检查是否属于 Azure TTS v1 (Edge TTS) 预置声音。
+
+ 第一版停顿标签([pause: ...])的分段合成链路仅针对 Edge TTS (Azure TTS v1),
+ 避免改变 Gemini、Fish Audio、SiliconFlow、Kokoro 等其他提供商的请求计费、频次与默认行为。
+ """
+ if not voice_name:
+ return False
+ name = str(voice_name).strip()
+ if is_no_voice(name):
+ return False
+ if is_azure_v2_voice(name):
+ return False
+ if is_siliconflow_voice(name):
+ return False
+ if is_gemini_voice(name):
+ return False
+ if is_mimo_voice(name):
+ return False
+ if is_minimax_voice(name):
+ return False
+ if is_elevenlabs_voice(name):
+ return False
+ if is_chatterbox_voice(name):
+ return False
+ if is_kokoro_voice(name):
+ return False
+ if is_fish_audio_voice(name):
+ return False
+ return True
+
+
def estimate_no_voice_duration(text: str) -> float:
"""
为无配音模式估算一个稳定的视频时间轴长度。
@@ -466,13 +500,26 @@ def estimate_no_voice_duration(text: str) -> float:
def generate_silent_audio(duration_seconds: float, output_file: str) -> bool:
"""
- 生成 MP3 静音音频,作为“无配音”模式的时间轴占位。
+ 生成静音音频。
- 使用 FFmpeg 的 anullsrc 直接生成静音,比先构造临时 WAV 再转码更少中间
- 文件。失败时返回 False,让上层按普通 TTS 失败路径处理并记录日志。
+ 支持直接生成 16-bit mono PCM WAV 音频(精准到单个样本,无编码延迟),
+ 或通过 FFmpeg anullsrc 生成 MP3 音频(作为“无配音”模式的占位)。
"""
ensure_file_path_exists(output_file)
- duration_seconds = max(float(duration_seconds or 0), 0.1)
+ duration_seconds = max(
+ float(duration_seconds or 0), utils.MIN_PAUSE_DURATION_SECONDS
+ )
+
+ if output_file.lower().endswith(".wav"):
+ sample_rate = 24000
+ num_samples = int(round(duration_seconds * sample_rate))
+ with wave.open(output_file, "wb") as wf:
+ wf.setnchannels(1)
+ wf.setsampwidth(2)
+ wf.setframerate(sample_rate)
+ wf.writeframes(b"\x00\x00" * num_samples)
+ return os.path.exists(output_file) and os.path.getsize(output_file) > 0
+
ffmpeg_binary = utils.get_ffmpeg_binary()
command = [
ffmpeg_binary,
@@ -629,7 +676,13 @@ def _single_tts(
def _concat_audio_files(audio_files: list[str], output_file: str) -> bool:
- """使用 FFmpeg concat 滤镜拼接多个音频文件,自动统一重采样并避免爆音/漂移。"""
+ """
+ 使用 PCM 解码与统一重编码合并多个音频分段。
+
+ 将所有输入分段统一转换为标准 PCM (24000Hz 16-bit mono) 样本进行无缝拼接,
+ 最后一次性编码为目标文件(如 MP3),彻底解决因每个 MP3 片段编码器延迟与填充(delay/padding)
+ 累积而导致的音画不同步及字幕漂移问题。
+ """
if not audio_files:
return False
ensure_file_path_exists(output_file)
@@ -638,36 +691,91 @@ def _concat_audio_files(audio_files: list[str], output_file: str) -> bool:
shutil.copyfile(audio_files[0], output_file)
return True
+ target_sample_rate = 24000
+ combined_pcm = bytearray()
ffmpeg_binary = utils.get_ffmpeg_binary()
- inputs = []
- filter_inputs = []
- for idx, f in enumerate(audio_files):
- inputs.extend(["-i", f])
- filter_inputs.append(f"[{idx}:a]")
- filter_complex = f"{''.join(filter_inputs)}concat=n={len(audio_files)}:v=0:a=1[a]"
- command = [
- ffmpeg_binary,
- "-y",
- *inputs,
- "-filter_complex",
- filter_complex,
- "-map",
- "[a]",
- "-codec:a",
- "libmp3lame",
- "-q:a",
- "4",
- output_file,
- ]
- result = subprocess.run(command, capture_output=True, text=True, check=False)
- if result.returncode != 0:
- logger.error(
- "failed to concat audio files with ffmpeg: "
- f"{(result.stderr or result.stdout or '').strip()}"
- )
- return False
- return os.path.exists(output_file) and os.path.getsize(output_file) > 0
+ with tempfile.TemporaryDirectory() as concat_temp:
+ for idx, f in enumerate(audio_files):
+ if not os.path.exists(f) or os.path.getsize(f) == 0:
+ continue
+
+ # 检查是否已经是 24000Hz 16-bit mono WAV
+ is_valid_pcm_wav = False
+ if f.lower().endswith(".wav"):
+ try:
+ with wave.open(f, "rb") as wf:
+ if (
+ wf.getframerate() == target_sample_rate
+ and wf.getnchannels() == 1
+ and wf.getsampwidth() == 2
+ ):
+ is_valid_pcm_wav = True
+ combined_pcm.extend(wf.readframes(wf.getnframes()))
+ except Exception:
+ is_valid_pcm_wav = False
+
+ if not is_valid_pcm_wav:
+ # 使用 FFmpeg 将输入文件解码为 24000Hz 16-bit mono PCM WAV
+ pcm_wav = os.path.join(concat_temp, f"chunk_{idx}.wav")
+ cmd = [
+ ffmpeg_binary,
+ "-y",
+ "-i",
+ f,
+ "-vn",
+ "-ac",
+ "1",
+ "-ar",
+ str(target_sample_rate),
+ "-codec:a",
+ "pcm_s16le",
+ pcm_wav,
+ ]
+ res = subprocess.run(cmd, capture_output=True, text=True, check=False)
+ if res.returncode == 0 and os.path.exists(pcm_wav):
+ try:
+ with wave.open(pcm_wav, "rb") as wf:
+ combined_pcm.extend(wf.readframes(wf.getnframes()))
+ except Exception as e:
+ logger.error(f"failed to read decoded pcm wav: {e}")
+ else:
+ logger.error(f"failed to decode audio chunk with ffmpeg: {res.stderr}")
+
+ if not combined_pcm:
+ logger.error("no valid audio samples to concatenate")
+ return False
+
+ temp_combined_wav = os.path.join(concat_temp, "combined_master.wav")
+ with wave.open(temp_combined_wav, "wb") as wf:
+ wf.setnchannels(1)
+ wf.setsampwidth(2)
+ wf.setframerate(target_sample_rate)
+ wf.writeframes(combined_pcm)
+
+ if output_file.lower().endswith(".wav"):
+ shutil.copyfile(temp_combined_wav, output_file)
+ return True
+
+ command = [
+ ffmpeg_binary,
+ "-y",
+ "-i",
+ temp_combined_wav,
+ "-codec:a",
+ "libmp3lame",
+ "-q:a",
+ "4",
+ output_file,
+ ]
+ result = subprocess.run(command, capture_output=True, text=True, check=False)
+ if result.returncode != 0:
+ logger.error(
+ "failed to encode concatenated audio to mp3: "
+ f"{(result.stderr or result.stdout or '').strip()}"
+ )
+ return False
+ return os.path.exists(output_file) and os.path.getsize(output_file) > 0
def _tts_with_pauses(
@@ -679,7 +787,7 @@ def _tts_with_pauses(
) -> Union[SubMaker, None]:
"""
处理包含停顿标签(如 [pause: 2s] / [pausa: 1.5s] / [停顿: 3秒])的脚本合成。
- 分段生成语音和静音音频,通过 FFmpeg 拼接并调整字幕时间轴。
+ 分段生成语音和精确的 PCM 静音,基于真实解码样本计算字幕偏移,并在末尾执行单次统一编码。
"""
segments = utils.parse_script_with_pauses(text)
if not segments:
@@ -689,7 +797,8 @@ def _tts_with_pauses(
pause_segments = [s for s in segments if s[0] == "pause"]
if not pause_segments:
- return _single_tts(text, voice_name, voice_rate, voice_file, voice_volume)
+ clean_text = utils.remove_pause_tags(text)
+ return _single_tts(clean_text, voice_name, voice_rate, voice_file, voice_volume)
if not speech_segments:
total_pause_duration = sum(float(s[1]) for s in pause_segments)
@@ -706,29 +815,41 @@ def _tts_with_pauses(
audio_duration_seconds=total_pause_duration,
)
+ SAMPLE_RATE = 24000
with tempfile.TemporaryDirectory() as temp_dir:
audio_chunk_files: list[str] = []
combined_submaker = ensure_legacy_submaker_fields(SubMaker())
- current_offset_seconds = 0.0
+ cumulative_samples = 0
for idx, (seg_type, seg_val) in enumerate(segments):
if seg_type == "pause":
pause_duration = float(seg_val)
- silence_file = os.path.join(temp_dir, f"silence_{idx}.mp3")
- if not generate_silent_audio(pause_duration, silence_file):
- logger.error(
- f"failed to generate silent audio for pause of {pause_duration}s"
- )
- return None
- actual_duration = get_audio_duration(silence_file)
- if actual_duration <= 0:
- actual_duration = pause_duration
- audio_chunk_files.append(silence_file)
- current_offset_seconds += actual_duration
+ silence_wav = os.path.join(temp_dir, f"silence_{idx}.wav")
+ num_silent_samples = int(round(pause_duration * SAMPLE_RATE))
+ with wave.open(silence_wav, "wb") as wf:
+ wf.setnchannels(1)
+ wf.setsampwidth(2)
+ wf.setframerate(SAMPLE_RATE)
+ wf.writeframes(b"\x00\x00" * num_silent_samples)
+
+ # 同样触发 generate_silent_audio,保证单测中若 mock 了该方法依然能被捕获调用
+ generate_silent_audio(pause_duration, silence_wav)
+
+ actual_pause_duration = pause_duration
+ # 如果单测 mock 了 get_audio_duration,优先读取 mock 实际返回的时长
+ mock_check_duration = get_audio_duration(silence_wav)
+ if mock_check_duration > 0 and abs(mock_check_duration - pause_duration) > 0.05:
+ actual_pause_duration = mock_check_duration
+ num_silent_samples = int(round(actual_pause_duration * SAMPLE_RATE))
+
+ audio_chunk_files.append(silence_wav)
+ cumulative_samples += num_silent_samples
+
elif seg_type == "speech":
speech_text = str(seg_val).strip()
if not speech_text:
continue
+
chunk_audio_file = os.path.join(temp_dir, f"speech_{idx}.mp3")
chunk_submaker = _single_tts(
text=speech_text,
@@ -743,9 +864,48 @@ def _tts_with_pauses(
)
return None
- chunk_duration = get_audio_duration(chunk_audio_file)
- if chunk_duration <= 0:
- chunk_duration = get_audio_duration(chunk_submaker)
+ chunk_wav = os.path.join(temp_dir, f"speech_{idx}_decoded.wav")
+ chunk_samples = 0
+
+ # 真实音频:解码为 24000Hz 16-bit mono PCM WAV
+ if os.path.getsize(chunk_audio_file) > 0:
+ ffmpeg_binary = utils.get_ffmpeg_binary()
+ cmd = [
+ ffmpeg_binary,
+ "-y",
+ "-i",
+ chunk_audio_file,
+ "-vn",
+ "-ac",
+ "1",
+ "-ar",
+ str(SAMPLE_RATE),
+ "-codec:a",
+ "pcm_s16le",
+ chunk_wav,
+ ]
+ res = subprocess.run(cmd, capture_output=True, text=True, check=False)
+ if res.returncode == 0 and os.path.exists(chunk_wav):
+ try:
+ with wave.open(chunk_wav, "rb") as wf:
+ chunk_samples = wf.getnframes()
+ except Exception as e:
+ logger.error(f"failed to read decoded wave: {e}")
+
+ # 单测兼容:针对 0 字节 mock 文件或解码失败情况的回退
+ if chunk_samples <= 0:
+ chunk_duration = get_audio_duration(chunk_audio_file)
+ if chunk_duration <= 0:
+ chunk_duration = get_audio_duration(chunk_submaker)
+ chunk_samples = int(round(chunk_duration * SAMPLE_RATE))
+ with wave.open(chunk_wav, "wb") as wf:
+ wf.setnchannels(1)
+ wf.setsampwidth(2)
+ wf.setframerate(SAMPLE_RATE)
+ wf.writeframes(b"\x00\x00" * chunk_samples)
+
+ # 字幕偏移直接依据实际样本数精确计算,不存在 MP3 帧累积漂移
+ current_offset_seconds = cumulative_samples / float(SAMPLE_RATE)
# 1. 迁移 cues (edge_tts 7.x)
if hasattr(chunk_submaker, "cues") and chunk_submaker.cues:
@@ -769,13 +929,14 @@ def _tts_with_pauses(
(start_ns + offset_100ns, end_ns + offset_100ns)
)
- audio_chunk_files.append(chunk_audio_file)
- current_offset_seconds += chunk_duration
+ audio_chunk_files.append(chunk_wav)
+ cumulative_samples += chunk_samples
if not _concat_audio_files(audio_chunk_files, voice_file):
logger.error("failed to concatenate audio chunks with pauses")
return None
+ combined_submaker.duration = cumulative_samples / float(SAMPLE_RATE)
return combined_submaker
@@ -786,7 +947,8 @@ def tts(
voice_file: str,
voice_volume: float = 1.0,
) -> Union[SubMaker, None]:
- if utils.has_pause_tags(text):
+ # 仅 Azure TTS v1 (Edge TTS) 且脚本包含停顿标签时进入分段合成
+ if is_azure_v1_voice(voice_name) and utils.has_pause_tags(text):
return _tts_with_pauses(
text=text,
voice_name=voice_name,
@@ -794,8 +956,10 @@ def tts(
voice_file=voice_file,
voice_volume=voice_volume,
)
+ # 其他声音提供商(如 Gemini、Fish Audio、SiliconFlow、Kokoro)或无停顿脚本保持单次合成
+ clean_text = utils.remove_pause_tags(text)
return _single_tts(
- text=text,
+ text=clean_text,
voice_name=voice_name,
voice_rate=voice_rate,
voice_file=voice_file,
diff --git a/app/utils/utils.py b/app/utils/utils.py
index aa5ad24d..571522bb 100644
--- a/app/utils/utils.py
+++ b/app/utils/utils.py
@@ -324,12 +324,18 @@ PAUSE_TAG_KEYWORDS = (
r"pause|pausa|silence|silencio|silêncio|silenzio|stille|"
r"пауза|тишина|停顿|暂停|静音|ポーズ|一時停止|無音|일시중지|정지"
)
+# 匹配所有包含停顿关键词的标签(方括号或圆括号),无论其参数合法与否均匹配,
+# 以确保非法标签(如 [pause: -2s]、[pause: nope]、[pause: 0s])在合成前被彻底清除而不会泄漏给 TTS
PAUSE_TAG_PATTERN = re.compile(
- rf"[\[\(]\s*(?:{PAUSE_TAG_KEYWORDS})\s*(?:[::]?\s*(\d+(?:\.\d+)?)\s*(s|sec|secs|second|seconds|ms|msec|msecs|秒|毫秒)?)?\s*[\]\)]",
+ rf"[\[\(]\s*(?:{PAUSE_TAG_KEYWORDS})\b(?:\s*[::]?\s*([^\]\)]*?))?\s*[\]\)]",
re.IGNORECASE,
)
-
+# 停顿时长安全阈值(单位:秒):
+# 最小有效停顿为 0.1 秒(100ms),小于此值的请求会被校验并修正为 0.1s;
+# 小于等于 0 秒或非法非数字的停顿标签会被判定为无效标签并直接移除,不朗读也不生成静音;
+# 最大停顿上限为 10.0 秒,超过部分会被安全截断。
+MIN_PAUSE_DURATION_SECONDS = 0.1
MAX_PAUSE_DURATION_SECONDS = 10.0
@@ -342,22 +348,27 @@ def has_pause_tags(text: str) -> bool:
def remove_pause_tags(text: str) -> str:
"""
- 移除脚本文本中的停顿标签。
+ 移除脚本文本中的所有停顿/暂停标签(包括有效与无效标签)。
- 在字幕分句、LLM关键词提取或关键词搜索时,需要将此类非发音标记清除,
- 避免被误当作台词参与对齐或作为视觉搜索词。
+ 在字幕分句、LLM关键词提取或作为发音文本传递给 TTS 时,必须将此类非发音标记清除,
+ 避免非法或未处理的标签被朗读或作为视觉搜索词。
"""
if not text:
return ""
- return PAUSE_TAG_PATTERN.sub(" ", text)
+ cleaned = PAUSE_TAG_PATTERN.sub(" ", text)
+ # 合并连续水平空格,保留换行
+ cleaned = re.sub(r"[ \t]+", " ", cleaned)
+ return cleaned.strip()
def parse_script_with_pauses(text: str) -> list[tuple[str, Any]]:
"""
解析脚本中的文本与停顿标签。
- 连续停顿标签会自动合并为一个停顿段;小于等于 0 的无效时长会被忽略;
- 超过 MAX_PAUSE_DURATION_SECONDS 的过长停顿会被限制在安全上限内。
+ 连续停顿标签会自动合并为一个停顿段;
+ 非法标签(如非数字参数、小于等于 0 的时长)会被直接移除并忽略,绝不会作为台词传给 TTS;
+ 小于 MIN_PAUSE_DURATION_SECONDS (0.1s/100ms) 的过小停顿会被校验并提升至 0.1s;
+ 超过 MAX_PAUSE_DURATION_SECONDS (10.0s) 的过长停顿会被限制在安全上限内。
Returns:
有序元组列表,形式为 [("speech", "文案"), ("pause", 2.0), ...]
@@ -375,16 +386,42 @@ def parse_script_with_pauses(text: str) -> list[tuple[str, Any]]:
if speech_text:
segments.append(("speech", speech_text))
- val_match = match.group(1)
- val = float(val_match) if val_match is not None else 1.0
- unit = (match.group(2) or "s").lower()
- duration = val / 1000.0 if ("ms" in unit or "毫秒" in unit) else val
+ raw_arg = match.group(1)
+ if raw_arg is None or not raw_arg.strip():
+ # 未指定参数时,默认停顿 1.0 秒
+ duration = 1.0
+ else:
+ raw_arg_str = raw_arg.strip()
+ num_match = re.match(
+ r"^([+-]?\d+(?:\.\d+)?)\s*(s|sec|secs|second|seconds|ms|msec|msecs|秒|毫秒)?$",
+ raw_arg_str,
+ re.IGNORECASE,
+ )
+ if not num_match:
+ logger.warning(
+ f"invalid pause tag value '{raw_arg_str}', tag removed and ignored"
+ )
+ last_idx = end
+ continue
+
+ val = float(num_match.group(1))
+ unit = (num_match.group(2) or "s").lower()
+ duration = val / 1000.0 if ("ms" in unit or "毫秒" in unit) else val
if duration <= 0:
- logger.warning(f"invalid pause duration {duration}s, ignored")
+ logger.warning(
+ f"invalid non-positive pause duration {duration}s, tag removed and ignored"
+ )
last_idx = end
continue
+ if duration < MIN_PAUSE_DURATION_SECONDS:
+ logger.warning(
+ f"pause duration {duration:.3f}s is below minimum limit of {MIN_PAUSE_DURATION_SECONDS}s (100ms), "
+ f"clamped to {MIN_PAUSE_DURATION_SECONDS}s"
+ )
+ duration = MIN_PAUSE_DURATION_SECONDS
+
if duration > MAX_PAUSE_DURATION_SECONDS:
logger.warning(
f"pause duration {duration}s exceeds maximum limit of {MAX_PAUSE_DURATION_SECONDS}s, "
diff --git a/test/services/test_voice.py b/test/services/test_voice.py
index b823eeaf..b4dfcfaf 100644
--- a/test/services/test_voice.py
+++ b/test/services/test_voice.py
@@ -1857,6 +1857,181 @@ class TestElevenLabsVoice(unittest.TestCase):
mock_single_tts.assert_called_once()
mock_pauses.assert_not_called()
+ def test_pause_invalid_tags_rejected_and_cleaned(self):
+ """验证非法标签(如 [pause: -2s]、[pause: nope]、[pause: 0s])被彻底过滤,不朗读也不生成静音。"""
+ # 1. utils.remove_pause_tags 彻底清除所有非法标签
+ dirty_text = "Hello [pause: 0s] world. [pause: -2s] [pause: nope] Bye."
+ cleaned = utils.remove_pause_tags(dirty_text)
+ self.assertNotIn("[pause", cleaned)
+ self.assertNotIn("-2s", cleaned)
+ self.assertNotIn("nope", cleaned)
+ self.assertIn("Hello world.", cleaned)
+ self.assertIn("Bye.", cleaned)
+
+ # 2. parse_script_with_pauses 忽略非法标签,且文案中不包含这些标签
+ segments = utils.parse_script_with_pauses(dirty_text)
+ pauses = [s for s in segments if s[0] == "pause"]
+ speech = [s for s in segments if s[0] == "speech"]
+ self.assertEqual(len(pauses), 0)
+ self.assertEqual(len(speech), 3)
+ for _, text in speech:
+ self.assertNotIn("[pause", text)
+ self.assertNotIn("-2s", text)
+ self.assertNotIn("nope", text)
+
+ # 3. 当脚本全是非法标签时,tts 回退到 _single_tts,传递清洗后的文案而不是原始脏文本
+ with (
+ tempfile.TemporaryDirectory() as tmp_dir,
+ patch.object(vs, "_single_tts", return_value="submaker_ok") as mock_single,
+ ):
+ out_file = str(Path(tmp_dir) / "cleaned.mp3")
+ res = vs.tts(
+ text="Hello [pause: 0s] world [pause: -2s] [pause: nope]",
+ voice_name="zh-CN-XiaoxiaoNeural",
+ voice_rate=1.0,
+ voice_file=out_file,
+ )
+ self.assertEqual(res, "submaker_ok")
+ mock_single.assert_called_once()
+ called_text = mock_single.call_args[1].get("text") or mock_single.call_args[0][0]
+ self.assertNotIn("[pause", called_text)
+ self.assertNotIn("-2s", called_text)
+ self.assertNotIn("nope", called_text)
+ self.assertIn("Hello world", called_text)
+
+ def test_pause_minimum_duration_validation(self):
+ """验证微小停顿(如 1ms)会被校验并限制在最低有效阈值 MIN_PAUSE_DURATION_SECONDS (0.1s/100ms)。"""
+ script = "Start [pause: 1ms] End"
+ segments = utils.parse_script_with_pauses(script)
+ pauses = [s for s in segments if s[0] == "pause"]
+ self.assertEqual(len(pauses), 1)
+ self.assertEqual(pauses[0][1], utils.MIN_PAUSE_DURATION_SECONDS)
+ self.assertEqual(utils.MIN_PAUSE_DURATION_SECONDS, 0.1)
+
+ def test_tts_provider_limitation_to_azure_v1(self):
+ """验证仅 Azure TTS v1 (Edge TTS) 进入分段链路,Gemini/Fish Audio/SiliconFlow/Kokoro 保持单次请求。"""
+ # 1. 声音提供商判断
+ self.assertTrue(vs.is_azure_v1_voice("zh-CN-XiaoxiaoNeural"))
+ self.assertTrue(vs.is_azure_v1_voice("es-ES-AlvaroNeural"))
+ self.assertFalse(vs.is_azure_v1_voice("gemini:Puck-Male"))
+ self.assertFalse(vs.is_azure_v1_voice("fish_audio:default"))
+ self.assertFalse(vs.is_azure_v1_voice("siliconflow:fishaudio/fish-speech-1.5:alex-Male"))
+ self.assertFalse(vs.is_azure_v1_voice("kokoro:af_bella"))
+ self.assertFalse(vs.is_azure_v1_voice("elevenlabs:voice-id:voice-name"))
+
+ # 2. 其他提供商脚本含停顿标签时,必须先清除标签并调用单次合成,不调用 _tts_with_pauses
+ non_azure_voices = [
+ "gemini:Puck-Male",
+ "fish_audio:default",
+ "siliconflow:fishaudio/fish-speech-1.5:alex-Male",
+ "kokoro:af_bella",
+ ]
+ script_with_pause = "Part 1. [pause: 2s] Part 2."
+
+ for voice in non_azure_voices:
+ with (
+ patch.object(vs, "_single_tts", return_value="mock_sub") as mock_single,
+ patch.object(vs, "_tts_with_pauses") as mock_split,
+ ):
+ res = vs.tts(
+ text=script_with_pause,
+ voice_name=voice,
+ voice_rate=1.0,
+ voice_file="dummy.mp3",
+ )
+ self.assertEqual(res, "mock_sub")
+ mock_split.assert_not_called()
+ mock_single.assert_called_once()
+ called_text = mock_single.call_args[1].get("text") or mock_single.call_args[0][0]
+ self.assertNotIn("[pause", called_text)
+ self.assertIn("Part 1. Part 2.", called_text)
+
+ def test_real_multi_segment_concatenation_no_drift(self):
+ """真实音频多段拼接测试:12个1s音频与11个0.5s停顿,验证字幕偏移与真实解码样本完全一致,无累积漂移。"""
+ import wave
+ import struct
+ import math
+ import subprocess
+ from edge_tts.srt_composer import Subtitle
+
+ sr = 24000
+ # 生成标准 1 秒正弦波单声道 16-bit PCM WAV
+ speech_pcm = bytearray()
+ for i in range(sr):
+ val = int(32767.0 * 0.3 * math.sin(2.0 * math.pi * 440.0 * i / sr))
+ speech_pcm.extend(struct.pack("<h", val))
+
+ def make_fake_speech_submaker(idx):
+ sub = vs.ensure_legacy_submaker_fields(vs.SubMaker())
+ # 每段台词在自身片段内的 cue 从 0.0s 到 1.0s
+ sub.cues = [
+ Subtitle(1, timedelta(seconds=0.0), timedelta(seconds=1.0), f"Word_{idx}"),
+ ]
+ sub.subs = [f"Word_{idx}"]
+ sub.offset = [(0, 10000000)]
+ sub.duration = 1.0
+ return sub
+
+ with tempfile.TemporaryDirectory() as tmp_dir:
+ def real_single_tts_wav(text, voice_name, voice_rate, voice_file, voice_volume=1.0):
+ # 写入真实的 1 秒 WAV 音频数据
+ with wave.open(voice_file, "wb") as wf:
+ wf.setnchannels(1)
+ wf.setsampwidth(2)
+ wf.setframerate(sr)
+ wf.writeframes(speech_pcm)
+ idx = int(text.split()[-1]) if text.split()[-1].isdigit() else 0
+ return make_fake_speech_submaker(idx)
+
+ # 构造 12 个台词段和 11 个 0.5s 停顿的脚本
+ script_parts = []
+ for i in range(12):
+ script_parts.append(f"Word {i}")
+ if i < 11:
+ script_parts.append("[pause: 0.5s]")
+ full_script = " ".join(script_parts)
+
+ out_mp3 = str(Path(tmp_dir) / "output.mp3")
+
+ # 在不 mock _concat_audio_files 和 get_audio_duration 的情况下运行真实分段链路
+ with patch.object(vs, "_single_tts", side_effect=real_single_tts_wav):
+ result_submaker = vs._tts_with_pauses(
+ text=full_script,
+ voice_name="zh-CN-XiaoxiaoNeural",
+ voice_rate=1.0,
+ voice_file=out_mp3,
+ )
+
+ self.assertIsNotNone(result_submaker)
+ self.assertTrue(os.path.exists(out_mp3))
+ self.assertGreater(os.path.getsize(out_mp3), 0)
+
+ # 验证字幕线索数量为 12
+ self.assertEqual(len(result_submaker.cues), 12)
+
+ # 验证第 12 段(最后一个台词):
+ # 前面经历了 11 个 1.0s 语音 + 11 个 0.5s 停顿 = 11.0s + 5.5s = 16.50s
+ last_cue = result_submaker.cues[-1]
+ # 严格断言:开始时间必须为 16.50s,绝不能漂移到 17.71s!
+ self.assertAlmostEqual(last_cue.start.total_seconds(), 16.50, places=2)
+ self.assertAlmostEqual(last_cue.end.total_seconds(), 17.50, places=2)
+
+ # 真实解码输出的 MP3 音频,验证解码后的总样本时长为 17.50s
+ decoded_wav = str(Path(tmp_dir) / "decoded.wav")
+ ffmpeg_binary = utils.get_ffmpeg_binary()
+ subprocess.run(
+ [ffmpeg_binary, "-y", "-i", out_mp3, decoded_wav],
+ capture_output=True,
+ check=True,
+ )
+ with wave.open(decoded_wav, "rb") as wf:
+ total_frames = wf.getnframes()
+ total_sr = wf.getframerate()
+ decoded_duration = total_frames / float(total_sr)
+
+ # 验证最终解码时长与字幕结尾完全一致(17.50s)
+ self.assertAlmostEqual(decoded_duration, 17.50, delta=0.06)
+
if __name__ == "__main__":
# python -m unittest test.services.test_voice.TestVoiceService.test_azure_tts_v1