Python
refactored tts handler
DeepBeepMeepcdd5e975ab7b
diff --git a/models/TTS/chatterbox_handler.py b/models/TTS/chatterbox_handler.pynew file mode 100644index 0000000..daf64be--- /dev/null+++ b/models/TTS/chatterbox_handler.py@@ -0,0 +1,211 @@+import os++import gradio as gr++from shared.utils import files_locator as fl+++_FALLBACK_SUPPORTED_LANGUAGES = {+ "ar": "Arabic",+ "da": "Danish",+ "de": "German",+ "el": "Greek",+ "en": "English",+ "es": "Spanish",+ "fi": "Finnish",+ "fr": "French",+ "he": "Hebrew",+ "hi": "Hindi",+ "it": "Italian",+ "ja": "Japanese",+ "ko": "Korean",+ "ms": "Malay",+ "nl": "Dutch",+ "no": "Norwegian",+ "pl": "Polish",+ "pt": "Portuguese",+ "ru": "Russian",+ "sv": "Swedish",+ "sw": "Swahili",+ "tr": "Turkish",+ "zh": "Chinese",+}++TTS_MONOLOGUE_PROMPT = (+ "You are a speechwriting assistant. Generate a single-speaker monologue "+ "for a text-to-speech model based on the user prompt. Output only the "+ "monologue text. Do not include explanations, bullet lists, or stage "+ "directions. Keep a consistent tone and point of view. Use natural, "+ "spoken sentences with clear punctuation for pauses. Aim for a short "+ "monologue (4-8 sentences) unless the prompt asks for a different length.\n\n"+ "Example:\n"+ "I never thought a small town would teach me so much about patience. "+ "Every morning the same faces pass the bakery window, and I know their "+ "stories without a word. The bell over the door rings, the coffee steams, "+ "and time slows down just enough to breathe. Some days I miss the noise of "+ "the city, but most days I am grateful for the quiet. It lets me hear "+ "myself think, and that has become its own kind of music."+)+++def _get_supported_languages() -> dict:+ try:+ from .chatterbox.mtl_tts import SUPPORTED_LANGUAGES+ except Exception:+ return _FALLBACK_SUPPORTED_LANGUAGES+ return SUPPORTED_LANGUAGES+++def _get_language_choices() -> list[tuple[str, str]]:+ languages = _get_supported_languages()+ return [+ (f"{name} ({code})", code)+ for code, name in sorted(languages.items(), key=lambda item: item[1])+ ]+++def _get_chatterbox_model_def():+ return {+ "audio_only": True,+ "image_outputs": False,+ "sliding_window": False,+ "guidance_max_phases": 0,+ "no_negative_prompt": True,+ "image_prompt_types_allowed": "",+ "profiles_dir": ["chatterbox"],+ "audio_guide_label": "Voice to Replicate",+ "model_modes": {+ "choices": _get_language_choices(),+ "default": "en",+ "label": "Language",+ },+ "any_audio_prompt": True,+ "chatterbox_controls": True,+ "text_prompt_enhancer_instructions": TTS_MONOLOGUE_PROMPT,+ }+++def _get_chatterbox_download_def():+ mandatory_files = [+ "ve.safetensors",+ "t3_mtl23ls_v2.safetensors",+ "s3gen.pt",+ "grapheme_mtl_merged_expanded_v1.json",+ "conds.pt",+ "Cangjie5_TC.json",+ ]+ return {+ "repoId": "ResembleAI/chatterbox",+ "sourceFolderList": [""],+ "targetFolderList": ["chatterbox"],+ "fileList": [mandatory_files],+ }+++class family_handler:+ @staticmethod+ def query_supported_types():+ return ["chatterbox"]++ @staticmethod+ def query_family_maps():+ return {}, {}++ @staticmethod+ def query_model_family():+ return "tts"++ @staticmethod+ def query_family_infos():+ return {"tts": (200, "TTS")}++ @staticmethod+ def register_lora_cli_args(parser, lora_root):+ parser.add_argument(+ "--lora-dir-tts",+ type=str,+ default=None,+ help=f"Path to a directory that contains TTS settings (default: {os.path.join(lora_root, 'tts')})",+ )++ @staticmethod+ def get_lora_dir(base_model_type, args, lora_root):+ return getattr(args, "lora_dir_tts", None) or os.path.join(lora_root, "tts")++ @staticmethod+ def query_model_def(base_model_type, model_def):+ return _get_chatterbox_model_def()++ @staticmethod+ def query_model_files(computeList, base_model_type, model_def=None):+ return _get_chatterbox_download_def()++ @staticmethod+ def load_model(+ model_filename,+ model_type,+ base_model_type,+ model_def,+ quantizeTransformer=False,+ text_encoder_quantization=None,+ dtype=None,+ VAE_dtype=None,+ mixed_precision_transformer=False,+ save_quantized=False,+ submodel_no_list=None,+ text_encoder_filename=None,+ profile=0,+ **kwargs,+ ):+ from .chatterbox.pipeline import ChatterboxPipeline++ ckpt_root = fl.get_download_location()+ pipeline = ChatterboxPipeline(ckpt_root=ckpt_root, device="cpu")+ pipe = {+ "ve": pipeline.model.ve,+ "s3gen": pipeline.model.s3gen,+ "t3": pipeline.model.t3,+ "conds": pipeline.model.conds,+ }+ return pipeline, pipe++ @staticmethod+ def fix_settings(base_model_type, settings_version, model_def, ui_defaults):+ if "alt_prompt" not in ui_defaults:+ ui_defaults["alt_prompt"] = ""++ defaults = {+ "audio_prompt_type": "A",+ "model_mode": "en",+ }+ for key, value in defaults.items():+ ui_defaults.setdefault(key, value)++ if settings_version < 2.44:+ ui_defaults["guidance_scale"] = 1.0++ @staticmethod+ def update_default_settings(base_model_type, model_def, ui_defaults):+ ui_defaults.update(+ {+ "audio_prompt_type": "A",+ "model_mode": "en",+ "repeat_generation": 1,+ "video_length": 0,+ "num_inference_steps": 0,+ "negative_prompt": "",+ "exaggeration": 0.5,+ "temperature": 0.8,+ "pace": 0.5,+ "guidance_scale": 1.0,+ "multi_prompts_gen_type": 2,+ }+ )++ @staticmethod+ def validate_generative_prompt(base_model_type, model_def, inputs, one_prompt):+ if len(one_prompt) > 300:+ gr.Info(+ "It is recommended to use a prompt that has less than 300 characters,"+ " otherwise you may get unexpected results."+ )diff --git a/models/TTS/heartmula_handler.py b/models/TTS/heartmula_handler.pynew file mode 100644index 0000000..8f206d9--- /dev/null+++ b/models/TTS/heartmula_handler.py@@ -0,0 +1,223 @@+import os++import torch++from shared.utils import files_locator as fl+++HEARTMULA_VERSION = "3B"+++def _get_heartmula_model_def():+ return {+ "audio_only": True,+ "image_outputs": False,+ "sliding_window": False,+ "guidance_max_phases": 1,+ "no_negative_prompt": True,+ "image_prompt_types_allowed": "",+ "supports_early_stop": True,+ "profiles_dir": ["heartmula_oss_3b"],+ "alt_prompt": {+ "label": "Keywords / Tags",+ "placeholder": "piano,happy,wedding",+ "lines": 2,+ },+ "text_prompt_enhancer_instructions": (+ "You are a lyric-writing assistant. Generate a clean song lyric prompt "+ "for a text-to-song model. Output only the lyric text with optional "+ "section headers in square brackets (e.g., [Verse], [Chorus], [Bridge], "+ "[Intro], [Outro]). Do not include explanations, bullet lists, or tags. "+ "Keep a consistent theme, POV, and rhyme or rhythm where natural. Use "+ "short lines that are easy to sing.\n\n"+ "Example:\n"+ "[Verse]\n"+ "Morning light through the window pane\n"+ "I hum a tune to chase the rain\n"+ "Steady steps on a quiet street\n"+ "Heart and rhythm, gentle beat\n"+ ),+ "duration_slider": {+ "label": "Duration of the Song (in seconds)",+ "min": 30,+ "max": 240,+ "increment": 0.1,+ "default": 120,+ },+ "top_k_slider": True,+ "heartmula_cfg_scale": 1.5,+ "heartmula_topk": 50,+ "heartmula_max_audio_length_ms": 120000,+ "heartmula_codec_guidance_scale": 1.25,+ "heartmula_codec_steps": 10,+ "heartmula_codec_version": "",+ "compile": False, # ["transformer", "transformer2"]+ }+++def _get_heartmula_download_def(model_def):+ codec_version = (model_def or {}).get("heartmula_codec_version", "")+ codec_suffix = f"_{codec_version}" if codec_version else ""+ repo_id = "DeepBeepMeep/TTS"+ gen_files = [+ "gen_config.json",+ "tokenizer.json",+ f"codec_config{codec_suffix}.json",+ f"HeartMula_codec{codec_suffix}.safetensors",+ ]+ return [+ {+ "repoId": repo_id,+ "sourceFolderList": ["HeartMula"],+ "fileList": [gen_files],+ },+ ]+++class family_handler:+ @staticmethod+ def query_supported_types():+ return ["heartmula_oss_3b"]++ @staticmethod+ def query_family_maps():+ return {}, {}++ @staticmethod+ def query_model_family():+ return "tts"++ @staticmethod+ def query_family_infos():+ return {"tts": (200, "TTS")}++ @staticmethod+ def register_lora_cli_args(parser, lora_root):+ parser.add_argument(+ "--lora-dir-tts",+ type=str,+ default=None,+ help=f"Path to a directory that contains TTS settings (default: {os.path.join(lora_root, 'tts')})",+ )++ @staticmethod+ def get_lora_dir(base_model_type, args, lora_root):+ return getattr(args, "lora_dir_tts", None) or os.path.join(lora_root, "tts")++ @staticmethod+ def query_model_def(base_model_type, model_def):+ return _get_heartmula_model_def()++ @staticmethod+ def query_model_files(computeList, base_model_type, model_def=None):+ return _get_heartmula_download_def(model_def or {})++ @staticmethod+ def load_model(+ model_filename,+ model_type,+ base_model_type,+ model_def,+ quantizeTransformer=False,+ text_encoder_quantization=None,+ dtype=None,+ VAE_dtype=None,+ mixed_precision_transformer=False,+ save_quantized=False,+ submodel_no_list=None,+ text_encoder_filename=None,+ profile=0,+ **kwargs,+ ):+ from .HeartMula.pipeline import HeartMuLaPipeline++ ckpt_root = fl.get_download_location()+ weights_candidate = None+ if isinstance(model_filename, (list, tuple)):+ if len(model_filename) > 0:+ weights_candidate = model_filename[0]+ else:+ weights_candidate = model_filename+ heartmula_weights_path = None+ if weights_candidate:+ heartmula_weights_path = fl.locate_file(+ weights_candidate, error_if_none=False+ )+ if heartmula_weights_path is None:+ heartmula_weights_path = weights_candidate+ pipeline = HeartMuLaPipeline(+ ckpt_root=ckpt_root,+ device=torch.device("cpu"),+ version=HEARTMULA_VERSION,+ VAE_dtype=VAE_dtype,+ heartmula_weights_path=heartmula_weights_path,+ cfg_scale=model_def.get("heartmula_cfg_scale", 1.5),+ topk=model_def.get("heartmula_topk", 50),+ max_audio_length_ms=model_def.get("heartmula_max_audio_length_ms", 120000),+ codec_steps=model_def.get("heartmula_codec_steps", 10),+ codec_guidance_scale=model_def.get("heartmula_codec_guidance_scale", 1.25),+ codec_version=model_def.get("heartmula_codec_version", ""),+ )++ pipeline.mula.decoder[0].layers._compile_me = False+ pipeline.mula.backbone.layers._compile_me = False+ pipe = {+ "transformer": pipeline.mula,+ "transformer2": pipeline.mula.decoder[0],+ "codec": pipeline.codec,+ }+ pipe = {+ "pipe": pipe,+ "coTenantsMap": {+ "transformer": ["transformer2"],+ "transformer2": ["transformer"],+ },+ }++ if int(profile) in (2, 4, 5):+ pipe["budgets"] = {"transformer2": 200}++ return pipeline, pipe++ @staticmethod+ def fix_settings(base_model_type, settings_version, model_def, ui_defaults):+ if "alt_prompt" not in ui_defaults:+ ui_defaults["alt_prompt"] = ""++ defaults = {+ "audio_prompt_type": "",+ }+ for key, value in defaults.items():+ ui_defaults.setdefault(key, value)++ if settings_version < 2.44:+ ui_defaults["guidance_scale"] = model_def.get("heartmula_cfg_scale", 1.5)+ ui_defaults["top_k"] = model_def.get("heartmula_topk", 50)++ @staticmethod+ def update_default_settings(base_model_type, model_def, ui_defaults):+ duration_def = model_def.get("duration_slider", {})+ ui_defaults.update(+ {+ "audio_prompt_type": "",+ "alt_prompt": "piano,happy,wedding",+ "repeat_generation": 1,+ "duration_seconds": duration_def.get("default", 120),+ "video_length": 0,+ "num_inference_steps": 0,+ "negative_prompt": "",+ "temperature": 1.0,+ "guidance_scale": model_def.get("heartmula_cfg_scale", 1.5),+ "top_k": model_def.get("heartmula_topk", 50),+ "multi_prompts_gen_type": 2,+ }+ )++ @staticmethod+ def validate_generative_prompt(base_model_type, model_def, inputs, one_prompt):+ alt_prompt = inputs.get("alt_prompt", "")+ if alt_prompt is None or len(str(alt_prompt).strip()) == 0:+ return "Keywords prompt cannot be empty for HeartMuLa."+ if inputs.get("audio_guide") is not None or inputs.get("audio_guide2") is not None:+ return "HeartMuLa does not support reference audio yet."+ return Nonediff --git a/models/TTS/qwen3_handler.py b/models/TTS/qwen3_handler.pynew file mode 100644index 0000000..2a5207e--- /dev/null+++ b/models/TTS/qwen3_handler.py@@ -0,0 +1,482 @@+import json+import os+from pathlib import Path+from typing import Optional++import torch++from shared.utils import files_locator as fl+++QWEN3_TTS_VARIANTS = {+ "qwen3_tts_customvoice": {+ "repo": "Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice",+ "config_file": "qwen3_tts_customvoice.json",+ },+ "qwen3_tts_voicedesign": {+ "repo": "Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign",+ "config_file": "qwen3_tts_voicedesign.json",+ },+ "qwen3_tts_base": {+ "repo": "Qwen/Qwen3-TTS-12Hz-1.7B-Base",+ "config_file": "qwen3_tts_base.json",+ },+}++QWEN3_TTS_GENERATION_CONFIG = "qwen3_tts_generation_config.json"+_QWEN3_CONFIG_DIR = Path(__file__).resolve().parent / "qwen3" / "configs"++QWEN3_TTS_TEXT_TOKENIZER_DIR = "qwen3_tts_text_tokenizer"+QWEN3_TTS_SPEECH_TOKENIZER_DIR = "qwen3_tts_tokenizer_12hz"+QWEN3_TTS_SPEECH_TOKENIZER_WEIGHTS = "qwen3_tts_tokenizer_12hz.safetensors"+QWEN3_TTS_REPO = "DeepBeepMeep/TTS"+QWEN3_TTS_TEXT_TOKENIZER_FILES = [+ "merges.txt",+ "vocab.json",+ "tokenizer_config.json",+ "preprocessor_config.json",+]+QWEN3_TTS_SPEECH_TOKENIZER_FILES = [+ "config.json",+ "configuration.json",+ "preprocessor_config.json",+ QWEN3_TTS_SPEECH_TOKENIZER_WEIGHTS,+]++QWEN3_TTS_LANG_FALLBACK = [+ "auto",+ "chinese",+ "english",+ "japanese",+ "korean",+ "german",+ "french",+ "russian",+ "portuguese",+ "spanish",+ "italian",+]+QWEN3_TTS_SPEAKER_FALLBACK = [+ "serena",+ "vivian",+ "uncle_fu",+ "ryan",+ "aiden",+ "ono_anna",+ "sohee",+ "eric",+ "dylan",+]+QWEN3_TTS_SPEAKER_META = {+ "vivian": {+ "style": "Bright, slightly edgy young female voice",+ "language": "Chinese",+ },+ "serena": {+ "style": "Warm, gentle young female voice",+ "language": "Chinese",+ },+ "uncle_fu": {+ "style": "Seasoned male voice with a low, mellow timbre",+ "language": "Chinese",+ },+ "dylan": {+ "style": "Youthful Beijing male voice with a clear, natural timbre",+ "language": "Chinese (Beijing Dialect)",+ },+ "eric": {+ "style": "Lively Chengdu male voice with a slightly husky brightness",+ "language": "Chinese (Sichuan Dialect)",+ },+ "ryan": {+ "style": "Dynamic male voice with strong rhythmic drive",+ "language": "English",+ },+ "aiden": {+ "style": "Sunny American male voice with a clear midrange",+ "language": "English",+ },+ "ono_anna": {+ "style": "Playful Japanese female voice with a light, nimble timbre",+ "language": "Japanese",+ },+ "sohee": {+ "style": "Warm Korean female voice with rich emotion",+ "language": "Korean",+ },+}+TTS_MONOLOGUE_PROMPT = (+ "You are a speechwriting assistant. Generate a single-speaker monologue "+ "for a text-to-speech model based on the user prompt. Output only the "+ "monologue text. Do not include explanations, bullet lists, or stage "+ "directions. Keep a consistent tone and point of view. Use natural, "+ "spoken sentences with clear punctuation for pauses. Aim for a short "+ "monologue (4-8 sentences) unless the prompt asks for a different length.\n\n"+ "Example:\n"+ "I never thought a small town would teach me so much about patience. "+ "Every morning the same faces pass the bakery window, and I know their "+ "stories without a word. The bell over the door rings, the coffee steams, "+ "and time slows down just enough to breathe. Some days I miss the noise of "+ "the city, but most days I am grateful for the quiet. It lets me hear "+ "myself think, and that has become its own kind of music."+)+QWEN3_TTS_DURATION_SLIDER = {+ "label": "Max duration (seconds)",+ "min": 1,+ "max": 240,+ "increment": 1,+ "default": 20,+}+++def _format_qwen3_label(value: str) -> str:+ return value.replace("_", " ").title()+++def _format_qwen3_speaker_label(name: str) -> str:+ label = _format_qwen3_label(name)+ meta = QWEN3_TTS_SPEAKER_META.get(name.lower())+ if not meta:+ return label+ parts = []+ style = meta.get("style", "")+ language = meta.get("language", "")+ if style:+ parts.append(style)+ if language:+ parts.append(language)+ if not parts:+ return label+ return f"{label} ({'; '.join(parts)})"+++def get_qwen3_config_path(base_model_type: str) -> Optional[str]:+ variant = QWEN3_TTS_VARIANTS.get(base_model_type)+ if variant is None:+ return None+ config_path = _QWEN3_CONFIG_DIR / variant["config_file"]+ return str(config_path) if config_path.is_file() else None+++def get_qwen3_generation_config_path() -> Optional[str]:+ config_path = _QWEN3_CONFIG_DIR / QWEN3_TTS_GENERATION_CONFIG+ return str(config_path) if config_path.is_file() else None+++def load_qwen3_config(base_model_type: str) -> Optional[dict]:+ config_path = get_qwen3_config_path(base_model_type)+ if not config_path:+ return None+ with open(config_path, "r", encoding="utf-8") as handle:+ return json.load(handle)+++def get_qwen3_languages(base_model_type: str) -> list[str]:+ config = load_qwen3_config(base_model_type)+ if config is None:+ return list(QWEN3_TTS_LANG_FALLBACK)+ lang_map = config.get("talker_config", {}).get("codec_language_id", {})+ languages = [name for name in lang_map.keys() if "dialect" not in name.lower()]+ languages = ["auto"] + sorted({name.lower() for name in languages})+ return languages+++def get_qwen3_speakers(base_model_type: str) -> list[str]:+ config = load_qwen3_config(base_model_type)+ if config is None:+ return list(QWEN3_TTS_SPEAKER_FALLBACK)+ speakers = list(config.get("talker_config", {}).get("spk_id", {}).keys())+ speakers = sorted({name.lower() for name in speakers})+ return speakers or list(QWEN3_TTS_SPEAKER_FALLBACK)+++def get_qwen3_language_choices(base_model_type: str) -> list[tuple[str, str]]:+ return [(_format_qwen3_label(lang), lang) for lang in get_qwen3_languages(base_model_type)]+++def get_qwen3_speaker_choices(base_model_type: str) -> list[tuple[str, str]]:+ return [(_format_qwen3_speaker_label(name), name) for name in get_qwen3_speakers(base_model_type)]+++def get_qwen3_model_def(base_model_type: str) -> dict:+ common = {+ "audio_only": True,+ "image_outputs": False,+ "sliding_window": False,+ "guidance_max_phases": 0,+ "no_negative_prompt": True,+ "image_prompt_types_allowed": "",+ "supports_early_stop": True,+ "profiles_dir": [base_model_type],+ "duration_slider": dict(QWEN3_TTS_DURATION_SLIDER),+ "top_k_slider": True,+ "text_prompt_enhancer_instructions": TTS_MONOLOGUE_PROMPT,+ "compile": False,+ "parent_model_type": "qwen3_tts_base",+ }+ if base_model_type == "qwen3_tts_customvoice":+ speakers = get_qwen3_speakers(base_model_type)+ default_speaker = speakers[0] if speakers else ""+ return {+ **common,+ "model_modes": {+ "choices": get_qwen3_speaker_choices(base_model_type),+ "default": default_speaker,+ "label": "Speaker",+ },+ "alt_prompt": {+ "label": "Instruction (optional)",+ "placeholder": "calm, friendly, slightly husky",+ "lines": 2,+ },+ }+ if base_model_type == "qwen3_tts_voicedesign":+ return {+ **common,+ "model_modes": {+ "choices": get_qwen3_language_choices(base_model_type),+ "default": "auto",+ "label": "Language",+ },+ "alt_prompt": {+ "label": "Voice instruction",+ "placeholder": "young female, warm tone, clear articulation",+ "lines": 2,+ },+ }+ if base_model_type == "qwen3_tts_base":+ return {+ **common,+ "model_modes": {+ "choices": get_qwen3_language_choices(base_model_type),+ "default": "auto",+ "label": "Language",+ },+ "alt_prompt": {+ "label": "Reference transcript (optional)",+ "placeholder": "Okay. Yeah. I respect you, but you blew it.",+ "lines": 3,+ },+ "any_audio_prompt": True,+ "audio_guide_label": "Reference voice",+ }+ return common+++def get_qwen3_duration_default() -> int:+ return int(QWEN3_TTS_DURATION_SLIDER.get("default", 20))+++def get_qwen3_download_def(base_model_type: str) -> list[dict]:+ return [+ {+ "repoId": QWEN3_TTS_REPO,+ "sourceFolderList": [QWEN3_TTS_TEXT_TOKENIZER_DIR],+ "fileList": [QWEN3_TTS_TEXT_TOKENIZER_FILES],+ },+ {+ "repoId": QWEN3_TTS_REPO,+ "sourceFolderList": [QWEN3_TTS_SPEECH_TOKENIZER_DIR],+ "fileList": [QWEN3_TTS_SPEECH_TOKENIZER_FILES],+ },+ ]+++class family_handler:+ @staticmethod+ def query_supported_types():+ return list(QWEN3_TTS_VARIANTS)++ @staticmethod+ def query_family_maps():+ return {}, {}++ @staticmethod+ def query_model_family():+ return "tts"++ @staticmethod+ def query_family_infos():+ return {"tts": (200, "TTS")}++ @staticmethod+ def register_lora_cli_args(parser, lora_root):+ parser.add_argument(+ "--lora-dir-tts",+ type=str,+ default=None,+ help=f"Path to a directory that contains TTS settings (default: {os.path.join(lora_root, 'tts')})",+ )++ @staticmethod+ def get_lora_dir(base_model_type, args, lora_root):+ return getattr(args, "lora_dir_tts", None) or os.path.join(lora_root, "tts")++ @staticmethod+ def query_model_def(base_model_type, model_def):+ return get_qwen3_model_def(base_model_type)++ @staticmethod+ def query_model_files(computeList, base_model_type, model_def=None):+ return get_qwen3_download_def(base_model_type)++ @staticmethod+ def load_model(+ model_filename,+ model_type,+ base_model_type,+ model_def,+ quantizeTransformer=False,+ text_encoder_quantization=None,+ dtype=None,+ VAE_dtype=None,+ mixed_precision_transformer=False,+ save_quantized=False,+ submodel_no_list=None,+ text_encoder_filename=None,+ profile=0,+ **kwargs,+ ):+ from .qwen3.pipeline import Qwen3TTSPipeline++ ckpt_root = fl.get_download_location()+ weights_candidate = None+ if isinstance(model_filename, (list, tuple)):+ if len(model_filename) > 0:+ weights_candidate = model_filename[0]+ else:+ weights_candidate = model_filename+ weights_path = None+ if weights_candidate:+ weights_path = fl.locate_file(weights_candidate, error_if_none=False)+ if weights_path is None:+ weights_path = weights_candidate++ pipeline = Qwen3TTSPipeline(+ model_weights_path=weights_path,+ base_model_type=base_model_type,+ ckpt_root=ckpt_root,+ device=torch.device("cpu"),+ )++ pipe = {"transformer": pipeline.model}+ if getattr(pipeline, "speech_tokenizer", None) is not None:+ pipe["speech_tokenizer"] = pipeline.speech_tokenizer.model+ return pipeline, pipe++ @staticmethod+ def fix_settings(base_model_type, settings_version, model_def, ui_defaults):+ if "alt_prompt" not in ui_defaults:+ ui_defaults["alt_prompt"] = ""++ if base_model_type == "qwen3_tts_customvoice":+ speakers = get_qwen3_speakers(base_model_type)+ defaults = {+ "audio_prompt_type": "",+ "model_mode": speakers[0] if speakers else "",+ }+ elif base_model_type == "qwen3_tts_voicedesign":+ defaults = {+ "audio_prompt_type": "",+ "model_mode": "auto",+ }+ elif base_model_type == "qwen3_tts_base":+ defaults = {+ "audio_prompt_type": "A",+ "model_mode": "auto",+ }+ else:+ defaults = {+ "audio_prompt_type": "",+ "model_mode": "auto",+ }+ for key, value in defaults.items():+ ui_defaults.setdefault(key, value)++ if settings_version < 2.44:+ if model_def.get("top_k_slider", False):+ ui_defaults["top_k"] = 50++ @staticmethod+ def update_default_settings(base_model_type, model_def, ui_defaults):+ if base_model_type == "qwen3_tts_customvoice":+ speakers = get_qwen3_speakers(base_model_type)+ default_speaker = speakers[0] if speakers else ""+ ui_defaults.update(+ {+ "audio_prompt_type": "",+ "model_mode": default_speaker,+ "alt_prompt": "",+ "duration_seconds": get_qwen3_duration_default(),+ "repeat_generation": 1,+ "video_length": 0,+ "num_inference_steps": 0,+ "negative_prompt": "",+ "temperature": 0.9,+ "top_k": 50,+ "multi_prompts_gen_type": 2,+ }+ )+ return++ if base_model_type == "qwen3_tts_voicedesign":+ ui_defaults.update(+ {+ "audio_prompt_type": "",+ "model_mode": "auto",+ "alt_prompt": "young female, warm tone, clear articulation",+ "duration_seconds": get_qwen3_duration_default(),+ "repeat_generation": 1,+ "video_length": 0,+ "num_inference_steps": 0,+ "negative_prompt": "",+ "temperature": 0.9,+ "top_k": 50,+ "multi_prompts_gen_type": 2,+ }+ )+ return++ if base_model_type == "qwen3_tts_base":+ ui_defaults.update(+ {+ "audio_prompt_type": "A",+ "model_mode": "auto",+ "alt_prompt": "",+ "duration_seconds": get_qwen3_duration_default(),+ "repeat_generation": 1,+ "video_length": 0,+ "num_inference_steps": 0,+ "negative_prompt": "",+ "temperature": 0.9,+ "top_k": 50,+ "multi_prompts_gen_type": 2,+ }+ )++ @staticmethod+ def validate_generative_prompt(base_model_type, model_def, inputs, one_prompt):+ if base_model_type == "qwen3_tts_customvoice":+ if one_prompt is None or len(str(one_prompt).strip()) == 0:+ return "Prompt text cannot be empty for Qwen3 CustomVoice."+ speaker = inputs.get("model_mode", "")+ if not speaker:+ return "Please select a speaker for Qwen3 CustomVoice."+ speakers = get_qwen3_speakers(base_model_type)+ if speaker.lower() not in speakers:+ return f"Unsupported speaker '{speaker}'."+ return None++ if base_model_type == "qwen3_tts_voicedesign":+ if one_prompt is None or len(str(one_prompt).strip()) == 0:+ return "Prompt text cannot be empty for Qwen3 VoiceDesign."+ return None++ if base_model_type == "qwen3_tts_base":+ if one_prompt is None or len(str(one_prompt).strip()) == 0:+ return "Prompt text cannot be empty for Qwen3 Base voice clone."+ if inputs.get("audio_guide") is None:+ return "Qwen3 Base requires a reference audio clip."+ return None++ return Nonediff --git a/models/TTS/yue_handler.py b/models/TTS/yue_handler.pynew file mode 100644index 0000000..40fa4cd--- /dev/null+++ b/models/TTS/yue_handler.py@@ -0,0 +1,267 @@+import os+++YUE_STAGE1_COT_REPO = "m-a-p/YuE-s1-7B-anneal-en-cot"+YUE_STAGE1_ICL_REPO = "m-a-p/YuE-s1-7B-anneal-en-icl"+YUE_STAGE2_REPO = "m-a-p/YuE-s2-1B-general"+YUE_STAGE1_FILES = [+ "config.json",+]+YUE_STAGE2_FILES = [+ "config.json",+]+++def _get_yue_model_def(model_def):+ use_audio_prompt = bool(model_def.get("yue_audio_prompt", False))+ yue_def = {+ "audio_only": True,+ "image_outputs": False,+ "sliding_window": False,+ "guidance_max_phases": 0,+ "no_negative_prompt": True,+ "image_prompt_types_allowed": "",+ "profiles_dir": ["yue"],+ "alt_prompt": {+ "label": "Genres / Tags",+ "placeholder": "pop, dreamy, warm vocal, female, nostalgic",+ "lines": 2,+ },+ "yue_max_new_tokens": 3000,+ "yue_run_n_segments": 2,+ "yue_stage2_batch_size": 4,+ "yue_segment_duration": 6,+ "yue_prompt_start_time": 0.0,+ "yue_prompt_end_time": 30.0,+ }+ if use_audio_prompt:+ yue_def.update(+ {+ "any_audio_prompt": True,+ "audio_prompt_choices": True,+ "audio_guide_label": "Vocal prompt",+ "audio_guide2_label": "Instrumental prompt",+ "audio_prompt_type_sources": {+ "selection": ["", "A", "AB"],+ "labels": {+ "": "Lyrics only",+ "A": "Mixed audio prompt",+ "AB": "Vocal + Instrumental prompts",+ },+ "letters_filter": "AB",+ "default": "",+ },+ }+ )+ return yue_def+++def _get_yue_download_def(model_def):+ use_audio_prompt = bool(model_def.get("yue_audio_prompt", False))+ stage1_repo = YUE_STAGE1_ICL_REPO if use_audio_prompt else YUE_STAGE1_COT_REPO+ stage1_folder = os.path.basename(stage1_repo)+ stage2_folder = os.path.basename(YUE_STAGE2_REPO)+ xcodec_root = "xcodec_mini_infer"+ xcodec_source_folders = [+ "final_ckpt",+ "decoders",+ "models",+ "modules",+ "quantization",+ "RepCodec",+ "descriptaudiocodec",+ "vocos",+ "semantic_ckpts/hf_1_325000",+ ]+ xcodec_files = [+ ["config.yaml", "ckpt_00360000.pth"],+ ["config.yaml", "decoder_131000.pth", "decoder_151000.pth"],+ [],+ [],+ [],+ [],+ [],+ [],+ [],+ ]+ return [+ {+ "repoId": stage1_repo,+ "sourceFolderList": [""],+ "targetFolderList": [stage1_folder],+ "fileList": [YUE_STAGE1_FILES],+ },+ {+ "repoId": YUE_STAGE2_REPO,+ "sourceFolderList": [""],+ "targetFolderList": [stage2_folder],+ "fileList": [YUE_STAGE2_FILES],+ },+ {+ "repoId": stage1_repo,+ "sourceFolderList": [""],+ "targetFolderList": ["mm_tokenizer_v0.2_hf"],+ "fileList": [["tokenizer.model"]],+ },+ {+ "repoId": "m-a-p/xcodec_mini_infer",+ "sourceFolderList": [""],+ "targetFolderList": [xcodec_root],+ "fileList": [["vocoder.py", "post_process_audio.py"]],+ },+ {+ "repoId": "m-a-p/xcodec_mini_infer",+ "sourceFolderList": xcodec_source_folders,+ "targetFolderList": [xcodec_root] * len(xcodec_source_folders),+ "fileList": xcodec_files,+ },+ ]+++class family_handler:+ @staticmethod+ def query_supported_types():+ return ["yue"]++ @staticmethod+ def query_family_maps():+ return {}, {}++ @staticmethod+ def query_model_family():+ return "tts"++ @staticmethod+ def query_family_infos():+ return {"tts": (200, "TTS")}++ @staticmethod+ def register_lora_cli_args(parser, lora_root):+ parser.add_argument(+ "--lora-dir-tts",+ type=str,+ default=None,+ help=f"Path to a directory that contains TTS settings (default: {os.path.join(lora_root, 'tts')})",+ )++ @staticmethod+ def get_lora_dir(base_model_type, args, lora_root):+ return getattr(args, "lora_dir_tts", None) or os.path.join(lora_root, "tts")++ @staticmethod+ def query_model_def(base_model_type, model_def):+ return _get_yue_model_def(model_def)++ @staticmethod+ def query_model_files(computeList, base_model_type, model_def=None):+ return _get_yue_download_def(model_def or {})++ @staticmethod+ def load_model(+ model_filename,+ model_type,+ base_model_type,+ model_def,+ quantizeTransformer=False,+ text_encoder_quantization=None,+ dtype=None,+ VAE_dtype=None,+ mixed_precision_transformer=False,+ save_quantized=False,+ submodel_no_list=None,+ text_encoder_filename=None,+ profile=0,+ **kwargs,+ ):+ from .yue.pipeline import YuePipeline++ if isinstance(model_filename, list):+ stage1_weights = model_filename[0] if len(model_filename) > 0 else ""+ stage2_weights = model_filename[1] if len(model_filename) > 1 else ""+ else:+ stage1_weights = model_filename or ""+ stage2_weights = ""++ pipeline = YuePipeline(+ stage1_weights_path=stage1_weights,+ stage2_weights_path=stage2_weights,+ use_audio_prompt=bool(model_def.get("yue_audio_prompt", False)),+ max_new_tokens=model_def.get("yue_max_new_tokens", 200),+ run_n_segments=model_def.get("yue_run_n_segments", 1),+ stage2_batch_size=model_def.get("yue_stage2_batch_size", 10),+ segment_duration=model_def.get("yue_segment_duration", 6),+ prompt_start_time=model_def.get("yue_prompt_start_time", 0.0),+ prompt_end_time=model_def.get("yue_prompt_end_time", 30.0),+ )++ pipe = {+ "transformer": pipeline.model_stage1,+ "transformer2": pipeline.model_stage2,+ "codec_model": pipeline.codec_model,+ "vocoder_vocal": pipeline.vocoder_vocal,+ "vocoder_inst": pipeline.vocoder_inst,+ }+ return pipeline, pipe++ @staticmethod+ def fix_settings(base_model_type, settings_version, model_def, ui_defaults):+ if "alt_prompt" not in ui_defaults:+ ui_defaults["alt_prompt"] = ""++ defaults = {+ "audio_prompt_type": "",+ }+ for key, value in defaults.items():+ ui_defaults.setdefault(key, value)++ @staticmethod+ def update_default_settings(base_model_type, model_def, ui_defaults):+ ui_defaults.update(+ {+ "audio_prompt_type": "",+ "alt_prompt": "pop, dreamy, warm vocal, female, nostalgic",+ "repeat_generation": 1,+ "video_length": 0,+ "num_inference_steps": 0,+ "negative_prompt": "",+ "temperature": 1.0,+ "multi_prompts_gen_type": 2,+ }+ )++ @staticmethod+ def validate_generative_prompt(base_model_type, model_def, inputs, one_prompt):+ if one_prompt is None or len(str(one_prompt).strip()) == 0:+ return "Lyrics prompt cannot be empty for Yue."+ alt_prompt = inputs.get("alt_prompt", "")+ if alt_prompt is None or len(str(alt_prompt).strip()) == 0:+ return "Genres prompt cannot be empty for Yue."+ audio_prompt_type = inputs.get("audio_prompt_type", "") or ""+ if model_def.get("yue_audio_prompt", False):+ if "A" in audio_prompt_type:+ if inputs.get("audio_guide") is None:+ return "You must provide a vocal or mixed audio prompt for Yue ICL."+ if "B" in audio_prompt_type and inputs.get("audio_guide2") is None:+ return "You must provide an instrumental prompt for Yue ICL."+ start_time = float(+ inputs.get(+ "yue_prompt_start_time",+ model_def.get("yue_prompt_start_time", 0.0),+ )+ )+ end_time = float(+ inputs.get(+ "yue_prompt_end_time",+ model_def.get("yue_prompt_end_time", 30.0),+ )+ )+ if start_time >= end_time:+ return "Audio prompt start time must be less than end time."+ if end_time - start_time > 30:+ return "Audio prompt duration should not exceed 30 seconds."+ elif inputs.get("audio_guide") is not None or inputs.get("audio_guide2") is not None:+ return "Select an audio prompt type for Yue ICL or clear audio prompts."+ else:+ if inputs.get("audio_guide") is not None or inputs.get("audio_guide2") is not None:+ return "Yue base model does not support audio prompts. Please use Yue ICL."+ return None