利用 AI 大模型和自动化工作流,根据主题或关键词一键生成高清短视频。Generate HD short videos from a topic or keyword with an automated AI workflow.
Python ai-video-generatorcontent-creationffmpeginstagram-reelsllmpythonshort-videosubtitlestext-to-speechtiktokvideo-automationvideo-workflowworkflow-automationyoutube-shorts
Add Xiaomi MiMo TTS provider
Harrybe2c8e270f11
diff --git a/app/services/voice.py b/app/services/voice.pyindex cff798e7..0d873785 100644--- a/app/services/voice.py+++ b/app/services/voice.py@@ -1,4 +1,6 @@import asyncio+import base64+import ioimport inspectimport mathimport os@@ -17,11 +19,14 @@ from edge_tts import SubMaker, submakerfrom loguru import loggerfrom moviepy.video.tools import subtitlesfrom moviepy.audio.io.AudioFileClip import AudioFileClip+from openai import OpenAIfrom app.config import configfrom app.utils import utils_DEFAULT_EDGE_TTS_TIMEOUT_SECONDS = 30.0+_MIMO_DEFAULT_BASE_URL = "https://api.xiaomimimo.com/v1"+_MIMO_DEFAULT_TTS_MODEL = "mimo-v2.5-tts"def _configure_pydub_ffmpeg(audio_segment_cls):@@ -111,6 +116,30 @@ def get_gemini_voices() -> list[str]:]+def get_mimo_voices() -> list[str]:+ """+ 获取 Xiaomi MiMo V2.5 TTS 的预置音色列表。++ 当前只接入官方文档里的 `mimo-v2.5-tts` 预置音色模式。音色设计+ `mimo-v2.5-tts-voicedesign` 和音色复刻 `mimo-v2.5-tts-voiceclone`+ 需要额外的输入表单和素材上传流程,先不混入普通 TTS 下拉框,避免+ 用户误以为选择一个 voice id 就能完成所有高级能力。+ """+ voices_with_gender = [+ ("mimo_default", "Female"),+ ("冰糖", "Female"),+ ("茉莉", "Female"),+ ("苏打", "Male"),+ ("白桦", "Male"),+ ("Mia", "Female"),+ ("Chloe", "Female"),+ ("Milo", "Male"),+ ("Dean", "Male"),+ ]++ return [f"mimo:{voice}-{gender}" for voice, gender in voices_with_gender]++def get_all_azure_voices(filter_locals=None) -> list[str]:azure_voices_str = """Name: af-ZA-AdriNeural@@ -1151,6 +1180,11 @@ def is_gemini_voice(voice_name: str):return voice_name.startswith("gemini:")+def is_mimo_voice(voice_name: str):+ """检查是否是 Xiaomi MiMo TTS 的声音"""+ return voice_name.startswith("mimo:")++def tts(text: str,voice_name: str,@@ -1189,6 +1223,18 @@ def tts(else:logger.error(f"Invalid gemini voice name format: {voice_name}")return None+ elif is_mimo_voice(voice_name):+ # 从voice_name中提取声音名称+ # 格式: mimo:voice-Gender;如果调用方已执行 parse_voice_name,+ # 则可能是 mimo:voice。两种格式都兼容。+ parts = voice_name.split(":")+ if len(parts) >= 2:+ voice_with_gender = parts[1]+ voice = voice_with_gender.split("-")[0]+ return mimo_tts(text, voice, voice_rate, voice_file, voice_volume)+ else:+ logger.error(f"Invalid mimo voice name format: {voice_name}")+ return Nonereturn azure_tts_v1(text, voice_name, voice_rate, voice_file)@@ -1872,6 +1918,106 @@ def gemini_tts(return None+def mimo_tts(+ text: str,+ voice_name: str,+ voice_rate: float,+ voice_file: str,+ voice_volume: float = 1.0,+) -> Union[SubMaker, None]:+ """+ 使用 Xiaomi MiMo V2.5 TTS 生成语音。++ 官方接口兼容 OpenAI Chat Completions,但 TTS 有两个关键差异:+ 1. 待合成文本必须放在 `assistant` 消息里;+ 2. 音频以 `message.audio.data` 的 base64 字符串返回。++ MiMo 当前没有返回逐词时间轴,因此这里复用项目已有的 legacy+ SubMaker 兜底方案:根据最终音频时长和脚本文本断句生成字幕时间轴。+ """+ from pydub import AudioSegment++ text = (text or "").strip()+ if not text:+ logger.error("MiMo TTS text is empty")+ return None++ api_key = config.app.get("mimo_api_key", "")+ if not api_key:+ logger.error("MiMo API key is not set")+ return None++ base_url = config.app.get("mimo_base_url", "") or _MIMO_DEFAULT_BASE_URL+ model_name = config.app.get("mimo_tts_model_name", "") or _MIMO_DEFAULT_TTS_MODEL+ style_prompt = config.app.get(+ "mimo_tts_style_prompt",+ "请用自然、清晰、适合短视频旁白的语气朗读。",+ )++ _configure_pydub_ffmpeg(AudioSegment)++ for i in range(3):+ try:+ logger.info(+ f"start mimo tts, model: {model_name}, voice: {voice_name}, try: {i + 1}"+ )+ ensure_file_path_exists(voice_file)++ client = OpenAI(api_key=api_key, base_url=base_url)+ completion = client.chat.completions.create(+ model=model_name,+ messages=[+ {"role": "user", "content": style_prompt},+ {"role": "assistant", "content": text},+ ],+ audio={+ "format": "wav",+ "voice": voice_name,+ },+ )++ if not completion or not getattr(completion, "choices", None):+ raise ValueError("MiMo TTS returned empty response")++ message = completion.choices[0].message+ audio = getattr(message, "audio", None)+ audio_data = None+ if isinstance(audio, dict):+ audio_data = audio.get("data")+ elif audio is not None:+ audio_data = getattr(audio, "data", None)++ if not audio_data:+ raise ValueError("MiMo TTS returned empty audio data")++ audio_bytes = base64.b64decode(audio_data)+ audio_segment = AudioSegment.from_file(io.BytesIO(audio_bytes), format="wav")++ output_format = utils.parse_extension(voice_file) or "mp3"+ if output_format == "wav":+ with open(voice_file, "wb") as f:+ f.write(audio_bytes)+ else:+ audio_segment.export(voice_file, format=output_format)++ audio_duration = len(audio_segment) / 1000.0+ sub_maker = ensure_legacy_submaker_fields(SubMaker())+ logger.success(f"mimo tts succeeded: {voice_file}")+ logger.debug(+ "mimo subtitle timeline generated, "+ f"duration: {audio_duration:.3f}s, output_format: {output_format}"+ )+ return populate_legacy_submaker_with_full_text(+ sub_maker=sub_maker,+ text=text,+ audio_duration_seconds=audio_duration,+ )+ except Exception as e:+ logger.error(f"mimo tts failed: {str(e)}")++ return None++def _format_text(text: str) -> str:# text = text.replace("\n", " ")text = text.replace("[", " ")diff --git a/config.example.toml b/config.example.tomlindex 061f6532..59a37b3c 100644--- a/config.example.toml+++ b/config.example.toml@@ -137,6 +137,10 @@ minimax_model_name = "MiniMax-M2.7"mimo_api_key = ""mimo_base_url = "https://api.xiaomimimo.com/v1"mimo_model_name = "mimo-v2.5-pro"+# Xiaomi MiMo TTS uses the same API key and base_url as MiMo LLM.+mimo_tts_model_name = "mimo-v2.5-tts"+# Optional natural language style prompt for MiMo TTS.+mimo_tts_style_prompt = "请用自然、清晰、适合短视频旁白的语气朗读。"########## DeepSeek API Key# Visit https://platform.deepseek.com/api_keys to get your API keydiff --git a/test/services/test_voice.py b/test/services/test_voice.pyindex 4116fd68..2c08ea8d 100644--- a/test/services/test_voice.py+++ b/test/services/test_voice.py@@ -1,4 +1,5 @@import asyncio+import base64import unittestimport osimport sys@@ -287,6 +288,97 @@ class TestVoiceService(unittest.TestCase):self.assertIn("Gemini subtitle generation should work now", subtitle_content)self.assertIn("Testing multiple lines", subtitle_content)+ def test_mimo_tts_uses_openai_compatible_audio_response(self):+ """+ 验证 Xiaomi MiMo TTS 可以消费 OpenAI-compatible 的音频响应结构。++ 这里用 fake OpenAI client 和 fake AudioSegment 覆盖真实网络与 ffmpeg,+ 确认运行时代码会把待合成文本放到 assistant message,并把返回的+ base64 WAV 音频导出到项目后续流程使用的音频文件。+ """++ class _FakeAudio:+ def __init__(self):+ self.data = base64.b64encode(b"RIFF-fake-wav").decode("utf-8")++ class _FakeMessage:+ def __init__(self):+ self.audio = _FakeAudio()++ class _FakeChoice:+ def __init__(self):+ self.message = _FakeMessage()++ class _FakeCompletion:+ def __init__(self):+ self.choices = [_FakeChoice()]++ class _FakeCompletions:+ def create(self, **kwargs):+ self.kwargs = kwargs+ return _FakeCompletion()++ class _FakeAudioSegment:+ def __len__(self):+ return 1800++ def export(self, output_file, format):+ Path(output_file).write_bytes(b"fake-mp3")++ fake_completions = _FakeCompletions()+ fake_client = SimpleNamespace(+ chat=SimpleNamespace(completions=fake_completions)+ )++ with tempfile.TemporaryDirectory() as tmp_dir, patch.object(+ vs,+ "OpenAI",+ return_value=fake_client,+ ) as openai_client, patch(+ "pydub.AudioSegment.from_file",+ return_value=_FakeAudioSegment(),+ ), patch.object(+ vs.config,+ "app",+ dict(+ vs.config.app,+ mimo_api_key="mimo-key",+ mimo_base_url="https://api.xiaomimimo.com/v1",+ mimo_tts_model_name="mimo-v2.5-tts",+ mimo_tts_style_prompt="用清晰的中文旁白朗读。",+ ),+ ):+ voice_file = str(Path(tmp_dir) / "mimo-tts.mp3")+ sub_maker = vs.mimo_tts(+ text="小米语音合成测试。第二句话。",+ voice_name="冰糖",+ voice_rate=1.0,+ voice_file=voice_file,+ voice_volume=1.0,+ )+ generated_audio = Path(voice_file).read_bytes()++ openai_client.assert_called_once_with(+ api_key="mimo-key",+ base_url="https://api.xiaomimimo.com/v1",+ )+ self.assertEqual(fake_completions.kwargs["model"], "mimo-v2.5-tts")+ self.assertEqual(+ fake_completions.kwargs["messages"],+ [+ {"role": "user", "content": "用清晰的中文旁白朗读。"},+ {"role": "assistant", "content": "小米语音合成测试。第二句话。"},+ ],+ )+ self.assertEqual(+ fake_completions.kwargs["audio"],+ {"format": "wav", "voice": "冰糖"},+ )+ self.assertEqual(generated_audio, b"fake-mp3")+ self.assertIsNotNone(sub_maker)+ self.assertEqual(getattr(sub_maker, "subs", []), ["小米语音合成测试", "第二句话"])+ self.assertEqual(len(getattr(sub_maker, "offset", [])), 2)+def test_generate_subtitle_keeps_edge_provider_for_gemini_legacy_submaker(self):"""验证 Gemini TTS 返回的 legacy 字幕结构在 edge provider 下可以直接产出diff --git a/webui/Main.py b/webui/Main.pyindex dee4405c..34188e63 100644--- a/webui/Main.py+++ b/webui/Main.py@@ -705,6 +705,7 @@ with middle_panel:("azure-tts-v2", "Azure TTS V2"),("siliconflow", "SiliconFlow TTS"),("gemini-tts", "Google Gemini TTS"),+ ("mimo-tts", "Xiaomi MiMo TTS"),]# 获取保存的TTS服务器,默认为v1@@ -734,6 +735,9 @@ with middle_panel:elif selected_tts_server == "gemini-tts":# 获取Gemini TTS的声音列表filtered_voices = voice.get_gemini_voices()+ elif selected_tts_server == "mimo-tts":+ # 获取 Xiaomi MiMo TTS 的预置音色列表+ filtered_voices = voice.get_mimo_voices()else:# 获取Azure的声音列表all_voices = voice.get_all_azure_voices(filter_locals=None)@@ -877,6 +881,32 @@ with middle_panel:config.siliconflow["api_key"] = siliconflow_api_key+ # 当选择 Xiaomi MiMo TTS 时,复用 MiMo LLM provider 的 API Key。+ # 这样用户如果同时使用 MiMo 生成文案和语音,只需要维护一份密钥。+ if selected_tts_server == "mimo-tts" or (+ voice_name and voice.is_mimo_voice(voice_name)+ ):+ saved_mimo_api_key = config.app.get("mimo_api_key", "")++ mimo_api_key = st.text_input(+ tr("MiMo API Key"),+ value=saved_mimo_api_key,+ type="password",+ key="mimo_tts_api_key_input",+ )++ st.info(+ tr("MiMo TTS Settings")+ + ":\n"+ + "- "+ + tr("Uses Xiaomi MiMo V2.5 TTS preset voices")+ + "\n"+ + "- "+ + tr("Speed and volume are currently handled by the provider defaults")+ )++ config.app["mimo_api_key"] = mimo_api_key+params.voice_volume = st.selectbox(tr("Speech Volume"),options=[0.6, 0.8, 1.0, 1.2, 1.5, 2.0, 3.0, 4.0, 5.0],diff --git a/webui/i18n/en.json b/webui/i18n/en.jsonindex d750f965..071f8821 100644--- a/webui/i18n/en.json+++ b/webui/i18n/en.json@@ -99,6 +99,10 @@"SiliconFlow TTS Settings": "SiliconFlow TTS Settings","Speed: Range [0.25, 4.0], default is 1.0": "Speed: Range [0.25, 4.0], default is 1.0","Volume: Uses Speech Volume setting, default 1.0 maps to gain 0": "Volume: Uses Speech Volume setting, default 1.0 maps to gain 0",+ "MiMo API Key": "Xiaomi MiMo API Key [Click to get](https://platform.xiaomimimo.com/docs/zh-CN/quick-start/first-api-call)",+ "MiMo TTS Settings": "Xiaomi MiMo TTS Settings",+ "Uses Xiaomi MiMo V2.5 TTS preset voices": "Uses Xiaomi MiMo V2.5 TTS preset voices",+ "Speed and volume are currently handled by the provider defaults": "Speed and volume are currently handled by the provider defaults","Hide Log": "Hide Log","Hide Basic Settings": "Hide Basic Settings\n\nHidden, the basic settings panel will not be displayed on the page.\n\nIf you need to display it again, please set `hide_config = false` in `config.toml`","LLM Settings": "**LLM Settings**",diff --git a/webui/i18n/zh.json b/webui/i18n/zh.jsonindex 7c2d90a4..896fc804 100644--- a/webui/i18n/zh.json+++ b/webui/i18n/zh.json@@ -99,6 +99,10 @@"SiliconFlow TTS Settings": "硅基流动TTS设置","Speed: Range [0.25, 4.0], default is 1.0": "语速范围 [0.25, 4.0],默认值为1.0","Volume: Uses Speech Volume setting, default 1.0 maps to gain 0": "音量:使用朗读音量设置,默认值1.0对应增益0",+ "MiMo API Key": "小米 MiMo API 密钥 [点击获取](https://platform.xiaomimimo.com/docs/zh-CN/quick-start/first-api-call)",+ "MiMo TTS Settings": "小米 MiMo TTS 设置",+ "Uses Xiaomi MiMo V2.5 TTS preset voices": "使用小米 MiMo V2.5 TTS 预置音色",+ "Speed and volume are currently handled by the provider defaults": "语速和音量当前使用服务商默认效果","Hide Log": "隐藏日志","Hide Basic Settings": "隐藏基础设置\n\n隐藏后,基础设置面板将不会显示在页面中。\n\n如需要再次显示,请在 `config.toml` 中设置 `hide_config = false`","LLM Settings": "**大模型设置**",