Skip to content
Python

refactored tts handler

DeepBeepMeepcdd5e975ab7b
diff --git a/models/TTS/chatterbox_handler.py b/models/TTS/chatterbox_handler.py
new file mode 100644
index 0000000..daf64be
--- /dev/null
+++ b/models/TTS/chatterbox_handler.py
@@ -0,0 +1,211 @@
+import os
+
+import gradio as gr
+
+from shared.utils import files_locator as fl
+
+
+_FALLBACK_SUPPORTED_LANGUAGES = {
+ "ar": "Arabic",
+ "da": "Danish",
+ "de": "German",
+ "el": "Greek",
+ "en": "English",
+ "es": "Spanish",
+ "fi": "Finnish",
+ "fr": "French",
+ "he": "Hebrew",
+ "hi": "Hindi",
+ "it": "Italian",
+ "ja": "Japanese",
+ "ko": "Korean",
+ "ms": "Malay",
+ "nl": "Dutch",
+ "no": "Norwegian",
+ "pl": "Polish",
+ "pt": "Portuguese",
+ "ru": "Russian",
+ "sv": "Swedish",
+ "sw": "Swahili",
+ "tr": "Turkish",
+ "zh": "Chinese",
+}
+
+TTS_MONOLOGUE_PROMPT = (
+ "You are a speechwriting assistant. Generate a single-speaker monologue "
+ "for a text-to-speech model based on the user prompt. Output only the "
+ "monologue text. Do not include explanations, bullet lists, or stage "
+ "directions. Keep a consistent tone and point of view. Use natural, "
+ "spoken sentences with clear punctuation for pauses. Aim for a short "
+ "monologue (4-8 sentences) unless the prompt asks for a different length.\n\n"
+ "Example:\n"
+ "I never thought a small town would teach me so much about patience. "
+ "Every morning the same faces pass the bakery window, and I know their "
+ "stories without a word. The bell over the door rings, the coffee steams, "
+ "and time slows down just enough to breathe. Some days I miss the noise of "
+ "the city, but most days I am grateful for the quiet. It lets me hear "
+ "myself think, and that has become its own kind of music."
+)
+
+
+def _get_supported_languages() -> dict:
+ try:
+ from .chatterbox.mtl_tts import SUPPORTED_LANGUAGES
+ except Exception:
+ return _FALLBACK_SUPPORTED_LANGUAGES
+ return SUPPORTED_LANGUAGES
+
+
+def _get_language_choices() -> list[tuple[str, str]]:
+ languages = _get_supported_languages()
+ return [
+ (f"{name} ({code})", code)
+ for code, name in sorted(languages.items(), key=lambda item: item[1])
+ ]
+
+
+def _get_chatterbox_model_def():
+ return {
+ "audio_only": True,
+ "image_outputs": False,
+ "sliding_window": False,
+ "guidance_max_phases": 0,
+ "no_negative_prompt": True,
+ "image_prompt_types_allowed": "",
+ "profiles_dir": ["chatterbox"],
+ "audio_guide_label": "Voice to Replicate",
+ "model_modes": {
+ "choices": _get_language_choices(),
+ "default": "en",
+ "label": "Language",
+ },
+ "any_audio_prompt": True,
+ "chatterbox_controls": True,
+ "text_prompt_enhancer_instructions": TTS_MONOLOGUE_PROMPT,
+ }
+
+
+def _get_chatterbox_download_def():
+ mandatory_files = [
+ "ve.safetensors",
+ "t3_mtl23ls_v2.safetensors",
+ "s3gen.pt",
+ "grapheme_mtl_merged_expanded_v1.json",
+ "conds.pt",
+ "Cangjie5_TC.json",
+ ]
+ return {
+ "repoId": "ResembleAI/chatterbox",
+ "sourceFolderList": [""],
+ "targetFolderList": ["chatterbox"],
+ "fileList": [mandatory_files],
+ }
+
+
+class family_handler:
+ @staticmethod
+ def query_supported_types():
+ return ["chatterbox"]
+
+ @staticmethod
+ def query_family_maps():
+ return {}, {}
+
+ @staticmethod
+ def query_model_family():
+ return "tts"
+
+ @staticmethod
+ def query_family_infos():
+ return {"tts": (200, "TTS")}
+
+ @staticmethod
+ def register_lora_cli_args(parser, lora_root):
+ parser.add_argument(
+ "--lora-dir-tts",
+ type=str,
+ default=None,
+ help=f"Path to a directory that contains TTS settings (default: {os.path.join(lora_root, 'tts')})",
+ )
+
+ @staticmethod
+ def get_lora_dir(base_model_type, args, lora_root):
+ return getattr(args, "lora_dir_tts", None) or os.path.join(lora_root, "tts")
+
+ @staticmethod
+ def query_model_def(base_model_type, model_def):
+ return _get_chatterbox_model_def()
+
+ @staticmethod
+ def query_model_files(computeList, base_model_type, model_def=None):
+ return _get_chatterbox_download_def()
+
+ @staticmethod
+ def load_model(
+ model_filename,
+ model_type,
+ base_model_type,
+ model_def,
+ quantizeTransformer=False,
+ text_encoder_quantization=None,
+ dtype=None,
+ VAE_dtype=None,
+ mixed_precision_transformer=False,
+ save_quantized=False,
+ submodel_no_list=None,
+ text_encoder_filename=None,
+ profile=0,
+ **kwargs,
+ ):
+ from .chatterbox.pipeline import ChatterboxPipeline
+
+ ckpt_root = fl.get_download_location()
+ pipeline = ChatterboxPipeline(ckpt_root=ckpt_root, device="cpu")
+ pipe = {
+ "ve": pipeline.model.ve,
+ "s3gen": pipeline.model.s3gen,
+ "t3": pipeline.model.t3,
+ "conds": pipeline.model.conds,
+ }
+ return pipeline, pipe
+
+ @staticmethod
+ def fix_settings(base_model_type, settings_version, model_def, ui_defaults):
+ if "alt_prompt" not in ui_defaults:
+ ui_defaults["alt_prompt"] = ""
+
+ defaults = {
+ "audio_prompt_type": "A",
+ "model_mode": "en",
+ }
+ for key, value in defaults.items():
+ ui_defaults.setdefault(key, value)
+
+ if settings_version < 2.44:
+ ui_defaults["guidance_scale"] = 1.0
+
+ @staticmethod
+ def update_default_settings(base_model_type, model_def, ui_defaults):
+ ui_defaults.update(
+ {
+ "audio_prompt_type": "A",
+ "model_mode": "en",
+ "repeat_generation": 1,
+ "video_length": 0,
+ "num_inference_steps": 0,
+ "negative_prompt": "",
+ "exaggeration": 0.5,
+ "temperature": 0.8,
+ "pace": 0.5,
+ "guidance_scale": 1.0,
+ "multi_prompts_gen_type": 2,
+ }
+ )
+
+ @staticmethod
+ def validate_generative_prompt(base_model_type, model_def, inputs, one_prompt):
+ if len(one_prompt) > 300:
+ gr.Info(
+ "It is recommended to use a prompt that has less than 300 characters,"
+ " otherwise you may get unexpected results."
+ )
diff --git a/models/TTS/heartmula_handler.py b/models/TTS/heartmula_handler.py
new file mode 100644
index 0000000..8f206d9
--- /dev/null
+++ b/models/TTS/heartmula_handler.py
@@ -0,0 +1,223 @@
+import os
+
+import torch
+
+from shared.utils import files_locator as fl
+
+
+HEARTMULA_VERSION = "3B"
+
+
+def _get_heartmula_model_def():
+ return {
+ "audio_only": True,
+ "image_outputs": False,
+ "sliding_window": False,
+ "guidance_max_phases": 1,
+ "no_negative_prompt": True,
+ "image_prompt_types_allowed": "",
+ "supports_early_stop": True,
+ "profiles_dir": ["heartmula_oss_3b"],
+ "alt_prompt": {
+ "label": "Keywords / Tags",
+ "placeholder": "piano,happy,wedding",
+ "lines": 2,
+ },
+ "text_prompt_enhancer_instructions": (
+ "You are a lyric-writing assistant. Generate a clean song lyric prompt "
+ "for a text-to-song model. Output only the lyric text with optional "
+ "section headers in square brackets (e.g., [Verse], [Chorus], [Bridge], "
+ "[Intro], [Outro]). Do not include explanations, bullet lists, or tags. "
+ "Keep a consistent theme, POV, and rhyme or rhythm where natural. Use "
+ "short lines that are easy to sing.\n\n"
+ "Example:\n"
+ "[Verse]\n"
+ "Morning light through the window pane\n"
+ "I hum a tune to chase the rain\n"
+ "Steady steps on a quiet street\n"
+ "Heart and rhythm, gentle beat\n"
+ ),
+ "duration_slider": {
+ "label": "Duration of the Song (in seconds)",
+ "min": 30,
+ "max": 240,
+ "increment": 0.1,
+ "default": 120,
+ },
+ "top_k_slider": True,
+ "heartmula_cfg_scale": 1.5,
+ "heartmula_topk": 50,
+ "heartmula_max_audio_length_ms": 120000,
+ "heartmula_codec_guidance_scale": 1.25,
+ "heartmula_codec_steps": 10,
+ "heartmula_codec_version": "",
+ "compile": False, # ["transformer", "transformer2"]
+ }
+
+
+def _get_heartmula_download_def(model_def):
+ codec_version = (model_def or {}).get("heartmula_codec_version", "")
+ codec_suffix = f"_{codec_version}" if codec_version else ""
+ repo_id = "DeepBeepMeep/TTS"
+ gen_files = [
+ "gen_config.json",
+ "tokenizer.json",
+ f"codec_config{codec_suffix}.json",
+ f"HeartMula_codec{codec_suffix}.safetensors",
+ ]
+ return [
+ {
+ "repoId": repo_id,
+ "sourceFolderList": ["HeartMula"],
+ "fileList": [gen_files],
+ },
+ ]
+
+
+class family_handler:
+ @staticmethod
+ def query_supported_types():
+ return ["heartmula_oss_3b"]
+
+ @staticmethod
+ def query_family_maps():
+ return {}, {}
+
+ @staticmethod
+ def query_model_family():
+ return "tts"
+
+ @staticmethod
+ def query_family_infos():
+ return {"tts": (200, "TTS")}
+
+ @staticmethod
+ def register_lora_cli_args(parser, lora_root):
+ parser.add_argument(
+ "--lora-dir-tts",
+ type=str,
+ default=None,
+ help=f"Path to a directory that contains TTS settings (default: {os.path.join(lora_root, 'tts')})",
+ )
+
+ @staticmethod
+ def get_lora_dir(base_model_type, args, lora_root):
+ return getattr(args, "lora_dir_tts", None) or os.path.join(lora_root, "tts")
+
+ @staticmethod
+ def query_model_def(base_model_type, model_def):
+ return _get_heartmula_model_def()
+
+ @staticmethod
+ def query_model_files(computeList, base_model_type, model_def=None):
+ return _get_heartmula_download_def(model_def or {})
+
+ @staticmethod
+ def load_model(
+ model_filename,
+ model_type,
+ base_model_type,
+ model_def,
+ quantizeTransformer=False,
+ text_encoder_quantization=None,
+ dtype=None,
+ VAE_dtype=None,
+ mixed_precision_transformer=False,
+ save_quantized=False,
+ submodel_no_list=None,
+ text_encoder_filename=None,
+ profile=0,
+ **kwargs,
+ ):
+ from .HeartMula.pipeline import HeartMuLaPipeline
+
+ ckpt_root = fl.get_download_location()
+ weights_candidate = None
+ if isinstance(model_filename, (list, tuple)):
+ if len(model_filename) > 0:
+ weights_candidate = model_filename[0]
+ else:
+ weights_candidate = model_filename
+ heartmula_weights_path = None
+ if weights_candidate:
+ heartmula_weights_path = fl.locate_file(
+ weights_candidate, error_if_none=False
+ )
+ if heartmula_weights_path is None:
+ heartmula_weights_path = weights_candidate
+ pipeline = HeartMuLaPipeline(
+ ckpt_root=ckpt_root,
+ device=torch.device("cpu"),
+ version=HEARTMULA_VERSION,
+ VAE_dtype=VAE_dtype,
+ heartmula_weights_path=heartmula_weights_path,
+ cfg_scale=model_def.get("heartmula_cfg_scale", 1.5),
+ topk=model_def.get("heartmula_topk", 50),
+ max_audio_length_ms=model_def.get("heartmula_max_audio_length_ms", 120000),
+ codec_steps=model_def.get("heartmula_codec_steps", 10),
+ codec_guidance_scale=model_def.get("heartmula_codec_guidance_scale", 1.25),
+ codec_version=model_def.get("heartmula_codec_version", ""),
+ )
+
+ pipeline.mula.decoder[0].layers._compile_me = False
+ pipeline.mula.backbone.layers._compile_me = False
+ pipe = {
+ "transformer": pipeline.mula,
+ "transformer2": pipeline.mula.decoder[0],
+ "codec": pipeline.codec,
+ }
+ pipe = {
+ "pipe": pipe,
+ "coTenantsMap": {
+ "transformer": ["transformer2"],
+ "transformer2": ["transformer"],
+ },
+ }
+
+ if int(profile) in (2, 4, 5):
+ pipe["budgets"] = {"transformer2": 200}
+
+ return pipeline, pipe
+
+ @staticmethod
+ def fix_settings(base_model_type, settings_version, model_def, ui_defaults):
+ if "alt_prompt" not in ui_defaults:
+ ui_defaults["alt_prompt"] = ""
+
+ defaults = {
+ "audio_prompt_type": "",
+ }
+ for key, value in defaults.items():
+ ui_defaults.setdefault(key, value)
+
+ if settings_version < 2.44:
+ ui_defaults["guidance_scale"] = model_def.get("heartmula_cfg_scale", 1.5)
+ ui_defaults["top_k"] = model_def.get("heartmula_topk", 50)
+
+ @staticmethod
+ def update_default_settings(base_model_type, model_def, ui_defaults):
+ duration_def = model_def.get("duration_slider", {})
+ ui_defaults.update(
+ {
+ "audio_prompt_type": "",
+ "alt_prompt": "piano,happy,wedding",
+ "repeat_generation": 1,
+ "duration_seconds": duration_def.get("default", 120),
+ "video_length": 0,
+ "num_inference_steps": 0,
+ "negative_prompt": "",
+ "temperature": 1.0,
+ "guidance_scale": model_def.get("heartmula_cfg_scale", 1.5),
+ "top_k": model_def.get("heartmula_topk", 50),
+ "multi_prompts_gen_type": 2,
+ }
+ )
+
+ @staticmethod
+ def validate_generative_prompt(base_model_type, model_def, inputs, one_prompt):
+ alt_prompt = inputs.get("alt_prompt", "")
+ if alt_prompt is None or len(str(alt_prompt).strip()) == 0:
+ return "Keywords prompt cannot be empty for HeartMuLa."
+ if inputs.get("audio_guide") is not None or inputs.get("audio_guide2") is not None:
+ return "HeartMuLa does not support reference audio yet."
+ return None
diff --git a/models/TTS/qwen3_handler.py b/models/TTS/qwen3_handler.py
new file mode 100644
index 0000000..2a5207e
--- /dev/null
+++ b/models/TTS/qwen3_handler.py
@@ -0,0 +1,482 @@
+import json
+import os
+from pathlib import Path
+from typing import Optional
+
+import torch
+
+from shared.utils import files_locator as fl
+
+
+QWEN3_TTS_VARIANTS = {
+ "qwen3_tts_customvoice": {
+ "repo": "Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice",
+ "config_file": "qwen3_tts_customvoice.json",
+ },
+ "qwen3_tts_voicedesign": {
+ "repo": "Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign",
+ "config_file": "qwen3_tts_voicedesign.json",
+ },
+ "qwen3_tts_base": {
+ "repo": "Qwen/Qwen3-TTS-12Hz-1.7B-Base",
+ "config_file": "qwen3_tts_base.json",
+ },
+}
+
+QWEN3_TTS_GENERATION_CONFIG = "qwen3_tts_generation_config.json"
+_QWEN3_CONFIG_DIR = Path(__file__).resolve().parent / "qwen3" / "configs"
+
+QWEN3_TTS_TEXT_TOKENIZER_DIR = "qwen3_tts_text_tokenizer"
+QWEN3_TTS_SPEECH_TOKENIZER_DIR = "qwen3_tts_tokenizer_12hz"
+QWEN3_TTS_SPEECH_TOKENIZER_WEIGHTS = "qwen3_tts_tokenizer_12hz.safetensors"
+QWEN3_TTS_REPO = "DeepBeepMeep/TTS"
+QWEN3_TTS_TEXT_TOKENIZER_FILES = [
+ "merges.txt",
+ "vocab.json",
+ "tokenizer_config.json",
+ "preprocessor_config.json",
+]
+QWEN3_TTS_SPEECH_TOKENIZER_FILES = [
+ "config.json",
+ "configuration.json",
+ "preprocessor_config.json",
+ QWEN3_TTS_SPEECH_TOKENIZER_WEIGHTS,
+]
+
+QWEN3_TTS_LANG_FALLBACK = [
+ "auto",
+ "chinese",
+ "english",
+ "japanese",
+ "korean",
+ "german",
+ "french",
+ "russian",
+ "portuguese",
+ "spanish",
+ "italian",
+]
+QWEN3_TTS_SPEAKER_FALLBACK = [
+ "serena",
+ "vivian",
+ "uncle_fu",
+ "ryan",
+ "aiden",
+ "ono_anna",
+ "sohee",
+ "eric",
+ "dylan",
+]
+QWEN3_TTS_SPEAKER_META = {
+ "vivian": {
+ "style": "Bright, slightly edgy young female voice",
+ "language": "Chinese",
+ },
+ "serena": {
+ "style": "Warm, gentle young female voice",
+ "language": "Chinese",
+ },
+ "uncle_fu": {
+ "style": "Seasoned male voice with a low, mellow timbre",
+ "language": "Chinese",
+ },
+ "dylan": {
+ "style": "Youthful Beijing male voice with a clear, natural timbre",
+ "language": "Chinese (Beijing Dialect)",
+ },
+ "eric": {
+ "style": "Lively Chengdu male voice with a slightly husky brightness",
+ "language": "Chinese (Sichuan Dialect)",
+ },
+ "ryan": {
+ "style": "Dynamic male voice with strong rhythmic drive",
+ "language": "English",
+ },
+ "aiden": {
+ "style": "Sunny American male voice with a clear midrange",
+ "language": "English",
+ },
+ "ono_anna": {
+ "style": "Playful Japanese female voice with a light, nimble timbre",
+ "language": "Japanese",
+ },
+ "sohee": {
+ "style": "Warm Korean female voice with rich emotion",
+ "language": "Korean",
+ },
+}
+TTS_MONOLOGUE_PROMPT = (
+ "You are a speechwriting assistant. Generate a single-speaker monologue "
+ "for a text-to-speech model based on the user prompt. Output only the "
+ "monologue text. Do not include explanations, bullet lists, or stage "
+ "directions. Keep a consistent tone and point of view. Use natural, "
+ "spoken sentences with clear punctuation for pauses. Aim for a short "
+ "monologue (4-8 sentences) unless the prompt asks for a different length.\n\n"
+ "Example:\n"
+ "I never thought a small town would teach me so much about patience. "
+ "Every morning the same faces pass the bakery window, and I know their "
+ "stories without a word. The bell over the door rings, the coffee steams, "
+ "and time slows down just enough to breathe. Some days I miss the noise of "
+ "the city, but most days I am grateful for the quiet. It lets me hear "
+ "myself think, and that has become its own kind of music."
+)
+QWEN3_TTS_DURATION_SLIDER = {
+ "label": "Max duration (seconds)",
+ "min": 1,
+ "max": 240,
+ "increment": 1,
+ "default": 20,
+}
+
+
+def _format_qwen3_label(value: str) -> str:
+ return value.replace("_", " ").title()
+
+
+def _format_qwen3_speaker_label(name: str) -> str:
+ label = _format_qwen3_label(name)
+ meta = QWEN3_TTS_SPEAKER_META.get(name.lower())
+ if not meta:
+ return label
+ parts = []
+ style = meta.get("style", "")
+ language = meta.get("language", "")
+ if style:
+ parts.append(style)
+ if language:
+ parts.append(language)
+ if not parts:
+ return label
+ return f"{label} ({'; '.join(parts)})"
+
+
+def get_qwen3_config_path(base_model_type: str) -> Optional[str]:
+ variant = QWEN3_TTS_VARIANTS.get(base_model_type)
+ if variant is None:
+ return None
+ config_path = _QWEN3_CONFIG_DIR / variant["config_file"]
+ return str(config_path) if config_path.is_file() else None
+
+
+def get_qwen3_generation_config_path() -> Optional[str]:
+ config_path = _QWEN3_CONFIG_DIR / QWEN3_TTS_GENERATION_CONFIG
+ return str(config_path) if config_path.is_file() else None
+
+
+def load_qwen3_config(base_model_type: str) -> Optional[dict]:
+ config_path = get_qwen3_config_path(base_model_type)
+ if not config_path:
+ return None
+ with open(config_path, "r", encoding="utf-8") as handle:
+ return json.load(handle)
+
+
+def get_qwen3_languages(base_model_type: str) -> list[str]:
+ config = load_qwen3_config(base_model_type)
+ if config is None:
+ return list(QWEN3_TTS_LANG_FALLBACK)
+ lang_map = config.get("talker_config", {}).get("codec_language_id", {})
+ languages = [name for name in lang_map.keys() if "dialect" not in name.lower()]
+ languages = ["auto"] + sorted({name.lower() for name in languages})
+ return languages
+
+
+def get_qwen3_speakers(base_model_type: str) -> list[str]:
+ config = load_qwen3_config(base_model_type)
+ if config is None:
+ return list(QWEN3_TTS_SPEAKER_FALLBACK)
+ speakers = list(config.get("talker_config", {}).get("spk_id", {}).keys())
+ speakers = sorted({name.lower() for name in speakers})
+ return speakers or list(QWEN3_TTS_SPEAKER_FALLBACK)
+
+
+def get_qwen3_language_choices(base_model_type: str) -> list[tuple[str, str]]:
+ return [(_format_qwen3_label(lang), lang) for lang in get_qwen3_languages(base_model_type)]
+
+
+def get_qwen3_speaker_choices(base_model_type: str) -> list[tuple[str, str]]:
+ return [(_format_qwen3_speaker_label(name), name) for name in get_qwen3_speakers(base_model_type)]
+
+
+def get_qwen3_model_def(base_model_type: str) -> dict:
+ common = {
+ "audio_only": True,
+ "image_outputs": False,
+ "sliding_window": False,
+ "guidance_max_phases": 0,
+ "no_negative_prompt": True,
+ "image_prompt_types_allowed": "",
+ "supports_early_stop": True,
+ "profiles_dir": [base_model_type],
+ "duration_slider": dict(QWEN3_TTS_DURATION_SLIDER),
+ "top_k_slider": True,
+ "text_prompt_enhancer_instructions": TTS_MONOLOGUE_PROMPT,
+ "compile": False,
+ "parent_model_type": "qwen3_tts_base",
+ }
+ if base_model_type == "qwen3_tts_customvoice":
+ speakers = get_qwen3_speakers(base_model_type)
+ default_speaker = speakers[0] if speakers else ""
+ return {
+ **common,
+ "model_modes": {
+ "choices": get_qwen3_speaker_choices(base_model_type),
+ "default": default_speaker,
+ "label": "Speaker",
+ },
+ "alt_prompt": {
+ "label": "Instruction (optional)",
+ "placeholder": "calm, friendly, slightly husky",
+ "lines": 2,
+ },
+ }
+ if base_model_type == "qwen3_tts_voicedesign":
+ return {
+ **common,
+ "model_modes": {
+ "choices": get_qwen3_language_choices(base_model_type),
+ "default": "auto",
+ "label": "Language",
+ },
+ "alt_prompt": {
+ "label": "Voice instruction",
+ "placeholder": "young female, warm tone, clear articulation",
+ "lines": 2,
+ },
+ }
+ if base_model_type == "qwen3_tts_base":
+ return {
+ **common,
+ "model_modes": {
+ "choices": get_qwen3_language_choices(base_model_type),
+ "default": "auto",
+ "label": "Language",
+ },
+ "alt_prompt": {
+ "label": "Reference transcript (optional)",
+ "placeholder": "Okay. Yeah. I respect you, but you blew it.",
+ "lines": 3,
+ },
+ "any_audio_prompt": True,
+ "audio_guide_label": "Reference voice",
+ }
+ return common
+
+
+def get_qwen3_duration_default() -> int:
+ return int(QWEN3_TTS_DURATION_SLIDER.get("default", 20))
+
+
+def get_qwen3_download_def(base_model_type: str) -> list[dict]:
+ return [
+ {
+ "repoId": QWEN3_TTS_REPO,
+ "sourceFolderList": [QWEN3_TTS_TEXT_TOKENIZER_DIR],
+ "fileList": [QWEN3_TTS_TEXT_TOKENIZER_FILES],
+ },
+ {
+ "repoId": QWEN3_TTS_REPO,
+ "sourceFolderList": [QWEN3_TTS_SPEECH_TOKENIZER_DIR],
+ "fileList": [QWEN3_TTS_SPEECH_TOKENIZER_FILES],
+ },
+ ]
+
+
+class family_handler:
+ @staticmethod
+ def query_supported_types():
+ return list(QWEN3_TTS_VARIANTS)
+
+ @staticmethod
+ def query_family_maps():
+ return {}, {}
+
+ @staticmethod
+ def query_model_family():
+ return "tts"
+
+ @staticmethod
+ def query_family_infos():
+ return {"tts": (200, "TTS")}
+
+ @staticmethod
+ def register_lora_cli_args(parser, lora_root):
+ parser.add_argument(
+ "--lora-dir-tts",
+ type=str,
+ default=None,
+ help=f"Path to a directory that contains TTS settings (default: {os.path.join(lora_root, 'tts')})",
+ )
+
+ @staticmethod
+ def get_lora_dir(base_model_type, args, lora_root):
+ return getattr(args, "lora_dir_tts", None) or os.path.join(lora_root, "tts")
+
+ @staticmethod
+ def query_model_def(base_model_type, model_def):
+ return get_qwen3_model_def(base_model_type)
+
+ @staticmethod
+ def query_model_files(computeList, base_model_type, model_def=None):
+ return get_qwen3_download_def(base_model_type)
+
+ @staticmethod
+ def load_model(
+ model_filename,
+ model_type,
+ base_model_type,
+ model_def,
+ quantizeTransformer=False,
+ text_encoder_quantization=None,
+ dtype=None,
+ VAE_dtype=None,
+ mixed_precision_transformer=False,
+ save_quantized=False,
+ submodel_no_list=None,
+ text_encoder_filename=None,
+ profile=0,
+ **kwargs,
+ ):
+ from .qwen3.pipeline import Qwen3TTSPipeline
+
+ ckpt_root = fl.get_download_location()
+ weights_candidate = None
+ if isinstance(model_filename, (list, tuple)):
+ if len(model_filename) > 0:
+ weights_candidate = model_filename[0]
+ else:
+ weights_candidate = model_filename
+ weights_path = None
+ if weights_candidate:
+ weights_path = fl.locate_file(weights_candidate, error_if_none=False)
+ if weights_path is None:
+ weights_path = weights_candidate
+
+ pipeline = Qwen3TTSPipeline(
+ model_weights_path=weights_path,
+ base_model_type=base_model_type,
+ ckpt_root=ckpt_root,
+ device=torch.device("cpu"),
+ )
+
+ pipe = {"transformer": pipeline.model}
+ if getattr(pipeline, "speech_tokenizer", None) is not None:
+ pipe["speech_tokenizer"] = pipeline.speech_tokenizer.model
+ return pipeline, pipe
+
+ @staticmethod
+ def fix_settings(base_model_type, settings_version, model_def, ui_defaults):
+ if "alt_prompt" not in ui_defaults:
+ ui_defaults["alt_prompt"] = ""
+
+ if base_model_type == "qwen3_tts_customvoice":
+ speakers = get_qwen3_speakers(base_model_type)
+ defaults = {
+ "audio_prompt_type": "",
+ "model_mode": speakers[0] if speakers else "",
+ }
+ elif base_model_type == "qwen3_tts_voicedesign":
+ defaults = {
+ "audio_prompt_type": "",
+ "model_mode": "auto",
+ }
+ elif base_model_type == "qwen3_tts_base":
+ defaults = {
+ "audio_prompt_type": "A",
+ "model_mode": "auto",
+ }
+ else:
+ defaults = {
+ "audio_prompt_type": "",
+ "model_mode": "auto",
+ }
+ for key, value in defaults.items():
+ ui_defaults.setdefault(key, value)
+
+ if settings_version < 2.44:
+ if model_def.get("top_k_slider", False):
+ ui_defaults["top_k"] = 50
+
+ @staticmethod
+ def update_default_settings(base_model_type, model_def, ui_defaults):
+ if base_model_type == "qwen3_tts_customvoice":
+ speakers = get_qwen3_speakers(base_model_type)
+ default_speaker = speakers[0] if speakers else ""
+ ui_defaults.update(
+ {
+ "audio_prompt_type": "",
+ "model_mode": default_speaker,
+ "alt_prompt": "",
+ "duration_seconds": get_qwen3_duration_default(),
+ "repeat_generation": 1,
+ "video_length": 0,
+ "num_inference_steps": 0,
+ "negative_prompt": "",
+ "temperature": 0.9,
+ "top_k": 50,
+ "multi_prompts_gen_type": 2,
+ }
+ )
+ return
+
+ if base_model_type == "qwen3_tts_voicedesign":
+ ui_defaults.update(
+ {
+ "audio_prompt_type": "",
+ "model_mode": "auto",
+ "alt_prompt": "young female, warm tone, clear articulation",
+ "duration_seconds": get_qwen3_duration_default(),
+ "repeat_generation": 1,
+ "video_length": 0,
+ "num_inference_steps": 0,
+ "negative_prompt": "",
+ "temperature": 0.9,
+ "top_k": 50,
+ "multi_prompts_gen_type": 2,
+ }
+ )
+ return
+
+ if base_model_type == "qwen3_tts_base":
+ ui_defaults.update(
+ {
+ "audio_prompt_type": "A",
+ "model_mode": "auto",
+ "alt_prompt": "",
+ "duration_seconds": get_qwen3_duration_default(),
+ "repeat_generation": 1,
+ "video_length": 0,
+ "num_inference_steps": 0,
+ "negative_prompt": "",
+ "temperature": 0.9,
+ "top_k": 50,
+ "multi_prompts_gen_type": 2,
+ }
+ )
+
+ @staticmethod
+ def validate_generative_prompt(base_model_type, model_def, inputs, one_prompt):
+ if base_model_type == "qwen3_tts_customvoice":
+ if one_prompt is None or len(str(one_prompt).strip()) == 0:
+ return "Prompt text cannot be empty for Qwen3 CustomVoice."
+ speaker = inputs.get("model_mode", "")
+ if not speaker:
+ return "Please select a speaker for Qwen3 CustomVoice."
+ speakers = get_qwen3_speakers(base_model_type)
+ if speaker.lower() not in speakers:
+ return f"Unsupported speaker '{speaker}'."
+ return None
+
+ if base_model_type == "qwen3_tts_voicedesign":
+ if one_prompt is None or len(str(one_prompt).strip()) == 0:
+ return "Prompt text cannot be empty for Qwen3 VoiceDesign."
+ return None
+
+ if base_model_type == "qwen3_tts_base":
+ if one_prompt is None or len(str(one_prompt).strip()) == 0:
+ return "Prompt text cannot be empty for Qwen3 Base voice clone."
+ if inputs.get("audio_guide") is None:
+ return "Qwen3 Base requires a reference audio clip."
+ return None
+
+ return None
diff --git a/models/TTS/yue_handler.py b/models/TTS/yue_handler.py
new file mode 100644
index 0000000..40fa4cd
--- /dev/null
+++ b/models/TTS/yue_handler.py
@@ -0,0 +1,267 @@
+import os
+
+
+YUE_STAGE1_COT_REPO = "m-a-p/YuE-s1-7B-anneal-en-cot"
+YUE_STAGE1_ICL_REPO = "m-a-p/YuE-s1-7B-anneal-en-icl"
+YUE_STAGE2_REPO = "m-a-p/YuE-s2-1B-general"
+YUE_STAGE1_FILES = [
+ "config.json",
+]
+YUE_STAGE2_FILES = [
+ "config.json",
+]
+
+
+def _get_yue_model_def(model_def):
+ use_audio_prompt = bool(model_def.get("yue_audio_prompt", False))
+ yue_def = {
+ "audio_only": True,
+ "image_outputs": False,
+ "sliding_window": False,
+ "guidance_max_phases": 0,
+ "no_negative_prompt": True,
+ "image_prompt_types_allowed": "",
+ "profiles_dir": ["yue"],
+ "alt_prompt": {
+ "label": "Genres / Tags",
+ "placeholder": "pop, dreamy, warm vocal, female, nostalgic",
+ "lines": 2,
+ },
+ "yue_max_new_tokens": 3000,
+ "yue_run_n_segments": 2,
+ "yue_stage2_batch_size": 4,
+ "yue_segment_duration": 6,
+ "yue_prompt_start_time": 0.0,
+ "yue_prompt_end_time": 30.0,
+ }
+ if use_audio_prompt:
+ yue_def.update(
+ {
+ "any_audio_prompt": True,
+ "audio_prompt_choices": True,
+ "audio_guide_label": "Vocal prompt",
+ "audio_guide2_label": "Instrumental prompt",
+ "audio_prompt_type_sources": {
+ "selection": ["", "A", "AB"],
+ "labels": {
+ "": "Lyrics only",
+ "A": "Mixed audio prompt",
+ "AB": "Vocal + Instrumental prompts",
+ },
+ "letters_filter": "AB",
+ "default": "",
+ },
+ }
+ )
+ return yue_def
+
+
+def _get_yue_download_def(model_def):
+ use_audio_prompt = bool(model_def.get("yue_audio_prompt", False))
+ stage1_repo = YUE_STAGE1_ICL_REPO if use_audio_prompt else YUE_STAGE1_COT_REPO
+ stage1_folder = os.path.basename(stage1_repo)
+ stage2_folder = os.path.basename(YUE_STAGE2_REPO)
+ xcodec_root = "xcodec_mini_infer"
+ xcodec_source_folders = [
+ "final_ckpt",
+ "decoders",
+ "models",
+ "modules",
+ "quantization",
+ "RepCodec",
+ "descriptaudiocodec",
+ "vocos",
+ "semantic_ckpts/hf_1_325000",
+ ]
+ xcodec_files = [
+ ["config.yaml", "ckpt_00360000.pth"],
+ ["config.yaml", "decoder_131000.pth", "decoder_151000.pth"],
+ [],
+ [],
+ [],
+ [],
+ [],
+ [],
+ [],
+ ]
+ return [
+ {
+ "repoId": stage1_repo,
+ "sourceFolderList": [""],
+ "targetFolderList": [stage1_folder],
+ "fileList": [YUE_STAGE1_FILES],
+ },
+ {
+ "repoId": YUE_STAGE2_REPO,
+ "sourceFolderList": [""],
+ "targetFolderList": [stage2_folder],
+ "fileList": [YUE_STAGE2_FILES],
+ },
+ {
+ "repoId": stage1_repo,
+ "sourceFolderList": [""],
+ "targetFolderList": ["mm_tokenizer_v0.2_hf"],
+ "fileList": [["tokenizer.model"]],
+ },
+ {
+ "repoId": "m-a-p/xcodec_mini_infer",
+ "sourceFolderList": [""],
+ "targetFolderList": [xcodec_root],
+ "fileList": [["vocoder.py", "post_process_audio.py"]],
+ },
+ {
+ "repoId": "m-a-p/xcodec_mini_infer",
+ "sourceFolderList": xcodec_source_folders,
+ "targetFolderList": [xcodec_root] * len(xcodec_source_folders),
+ "fileList": xcodec_files,
+ },
+ ]
+
+
+class family_handler:
+ @staticmethod
+ def query_supported_types():
+ return ["yue"]
+
+ @staticmethod
+ def query_family_maps():
+ return {}, {}
+
+ @staticmethod
+ def query_model_family():
+ return "tts"
+
+ @staticmethod
+ def query_family_infos():
+ return {"tts": (200, "TTS")}
+
+ @staticmethod
+ def register_lora_cli_args(parser, lora_root):
+ parser.add_argument(
+ "--lora-dir-tts",
+ type=str,
+ default=None,
+ help=f"Path to a directory that contains TTS settings (default: {os.path.join(lora_root, 'tts')})",
+ )
+
+ @staticmethod
+ def get_lora_dir(base_model_type, args, lora_root):
+ return getattr(args, "lora_dir_tts", None) or os.path.join(lora_root, "tts")
+
+ @staticmethod
+ def query_model_def(base_model_type, model_def):
+ return _get_yue_model_def(model_def)
+
+ @staticmethod
+ def query_model_files(computeList, base_model_type, model_def=None):
+ return _get_yue_download_def(model_def or {})
+
+ @staticmethod
+ def load_model(
+ model_filename,
+ model_type,
+ base_model_type,
+ model_def,
+ quantizeTransformer=False,
+ text_encoder_quantization=None,
+ dtype=None,
+ VAE_dtype=None,
+ mixed_precision_transformer=False,
+ save_quantized=False,
+ submodel_no_list=None,
+ text_encoder_filename=None,
+ profile=0,
+ **kwargs,
+ ):
+ from .yue.pipeline import YuePipeline
+
+ if isinstance(model_filename, list):
+ stage1_weights = model_filename[0] if len(model_filename) > 0 else ""
+ stage2_weights = model_filename[1] if len(model_filename) > 1 else ""
+ else:
+ stage1_weights = model_filename or ""
+ stage2_weights = ""
+
+ pipeline = YuePipeline(
+ stage1_weights_path=stage1_weights,
+ stage2_weights_path=stage2_weights,
+ use_audio_prompt=bool(model_def.get("yue_audio_prompt", False)),
+ max_new_tokens=model_def.get("yue_max_new_tokens", 200),
+ run_n_segments=model_def.get("yue_run_n_segments", 1),
+ stage2_batch_size=model_def.get("yue_stage2_batch_size", 10),
+ segment_duration=model_def.get("yue_segment_duration", 6),
+ prompt_start_time=model_def.get("yue_prompt_start_time", 0.0),
+ prompt_end_time=model_def.get("yue_prompt_end_time", 30.0),
+ )
+
+ pipe = {
+ "transformer": pipeline.model_stage1,
+ "transformer2": pipeline.model_stage2,
+ "codec_model": pipeline.codec_model,
+ "vocoder_vocal": pipeline.vocoder_vocal,
+ "vocoder_inst": pipeline.vocoder_inst,
+ }
+ return pipeline, pipe
+
+ @staticmethod
+ def fix_settings(base_model_type, settings_version, model_def, ui_defaults):
+ if "alt_prompt" not in ui_defaults:
+ ui_defaults["alt_prompt"] = ""
+
+ defaults = {
+ "audio_prompt_type": "",
+ }
+ for key, value in defaults.items():
+ ui_defaults.setdefault(key, value)
+
+ @staticmethod
+ def update_default_settings(base_model_type, model_def, ui_defaults):
+ ui_defaults.update(
+ {
+ "audio_prompt_type": "",
+ "alt_prompt": "pop, dreamy, warm vocal, female, nostalgic",
+ "repeat_generation": 1,
+ "video_length": 0,
+ "num_inference_steps": 0,
+ "negative_prompt": "",
+ "temperature": 1.0,
+ "multi_prompts_gen_type": 2,
+ }
+ )
+
+ @staticmethod
+ def validate_generative_prompt(base_model_type, model_def, inputs, one_prompt):
+ if one_prompt is None or len(str(one_prompt).strip()) == 0:
+ return "Lyrics prompt cannot be empty for Yue."
+ alt_prompt = inputs.get("alt_prompt", "")
+ if alt_prompt is None or len(str(alt_prompt).strip()) == 0:
+ return "Genres prompt cannot be empty for Yue."
+ audio_prompt_type = inputs.get("audio_prompt_type", "") or ""
+ if model_def.get("yue_audio_prompt", False):
+ if "A" in audio_prompt_type:
+ if inputs.get("audio_guide") is None:
+ return "You must provide a vocal or mixed audio prompt for Yue ICL."
+ if "B" in audio_prompt_type and inputs.get("audio_guide2") is None:
+ return "You must provide an instrumental prompt for Yue ICL."
+ start_time = float(
+ inputs.get(
+ "yue_prompt_start_time",
+ model_def.get("yue_prompt_start_time", 0.0),
+ )
+ )
+ end_time = float(
+ inputs.get(
+ "yue_prompt_end_time",
+ model_def.get("yue_prompt_end_time", 30.0),
+ )
+ )
+ if start_time >= end_time:
+ return "Audio prompt start time must be less than end time."
+ if end_time - start_time > 30:
+ return "Audio prompt duration should not exceed 30 seconds."
+ elif inputs.get("audio_guide") is not None or inputs.get("audio_guide2") is not None:
+ return "Select an audio prompt type for Yue ICL or clear audio prompts."
+ else:
+ if inputs.get("audio_guide") is not None or inputs.get("audio_guide2") is not None:
+ return "Yue base model does not support audio prompts. Please use Yue ICL."
+ return None