Skip to content
Python
9.1 KB
Raw Blame
import os


YUE_STAGE1_COT_REPO = "m-a-p/YuE-s1-7B-anneal-en-cot"
YUE_STAGE1_ICL_REPO = "m-a-p/YuE-s1-7B-anneal-en-icl"
YUE_STAGE2_REPO = "m-a-p/YuE-s2-1B-general"
YUE_STAGE1_FILES = [
    "config.json",
]
YUE_STAGE2_FILES = [
    "config.json",
]


def _get_yue_model_def(model_def):
    use_audio_prompt = bool(model_def.get("yue_audio_prompt", False))
    yue_def = {
        "audio_only": True,
        "image_outputs": False,
        "sliding_window": False,
        "guidance_max_phases": 0,
        "no_negative_prompt": True,
        "inference_steps": False,
        "temperature": True,
        "image_prompt_types_allowed": "",
        "profiles_dir": ["yue"],
        "alt_prompt": {
            "label": "Genres / Tags",
            "placeholder": "pop, dreamy, warm vocal, female, nostalgic",
            "lines": 2,
        },
        "yue_max_new_tokens": 3000,
        "yue_run_n_segments": 2,
        "yue_stage2_batch_size": 4,
        "yue_segment_duration": 6,
        "yue_prompt_start_time": 0.0,
        "yue_prompt_end_time": 30.0,
    }
    if use_audio_prompt:
        yue_def.update(
            {
                "any_audio_prompt": True,
                "audio_prompt_choices": True,
                "audio_guide_label": "Vocal prompt",
                "audio_guide2_label": "Instrumental prompt",
                "audio_prompt_type_sources": {
                    "selection": ["", "A", "AB"],
                    "labels": {
                        "": "Lyrics only",
                        "A": "Mixed audio prompt",
                        "AB": "Vocal + Instrumental prompts",
                    },
                    "letters_filter": "AB",
                    "default": "",
                },
            }
        )
    return yue_def


def _get_yue_download_def(model_def):
    use_audio_prompt = bool(model_def.get("yue_audio_prompt", False))
    stage1_repo = YUE_STAGE1_ICL_REPO if use_audio_prompt else YUE_STAGE1_COT_REPO
    stage1_folder = os.path.basename(stage1_repo)
    stage2_folder = os.path.basename(YUE_STAGE2_REPO)
    xcodec_root = "xcodec_mini_infer"
    xcodec_source_folders = [
        "final_ckpt",
        "decoders",
        "models",
        "modules",
        "quantization",
        "RepCodec",
        "descriptaudiocodec",
        "vocos",
        "semantic_ckpts/hf_1_325000",
    ]
    xcodec_files = [
        ["config.yaml", "ckpt_00360000.pth"],
        ["config.yaml", "decoder_131000.pth", "decoder_151000.pth"],
        [],
        [],
        [],
        [],
        [],
        [],
        [],
    ]
    return [
        {
            "repoId": stage1_repo,
            "sourceFolderList": [""],
            "targetFolderList": [stage1_folder],
            "fileList": [YUE_STAGE1_FILES],
        },
        {
            "repoId": YUE_STAGE2_REPO,
            "sourceFolderList": [""],
            "targetFolderList": [stage2_folder],
            "fileList": [YUE_STAGE2_FILES],
        },
        {
            "repoId": stage1_repo,
            "sourceFolderList": [""],
            "targetFolderList": ["mm_tokenizer_v0.2_hf"],
            "fileList": [["tokenizer.model"]],
        },
        {
            "repoId": "m-a-p/xcodec_mini_infer",
            "sourceFolderList": [""],
            "targetFolderList": [xcodec_root],
            "fileList": [["vocoder.py", "post_process_audio.py"]],
        },
        {
            "repoId": "m-a-p/xcodec_mini_infer",
            "sourceFolderList": xcodec_source_folders,
            "targetFolderList": [xcodec_root] * len(xcodec_source_folders),
            "fileList": xcodec_files,
        },
    ]


class family_handler:
    @staticmethod
    def query_supported_types():
        return ["yue"]

    @staticmethod
    def query_family_maps():
        return {}, {}

    @staticmethod
    def query_model_family():
        return "tts"

    @staticmethod
    def query_family_infos():
        return {"tts": (200, "TTS")}

    @staticmethod
    def register_lora_cli_args(parser, lora_root):
        parser.add_argument(
            "--lora-dir-tts",
            type=str,
            default=None,
            help=f"Path to a directory that contains TTS settings (default: {os.path.join(lora_root, 'tts')})",
        )

    @staticmethod
    def get_lora_dir(base_model_type, args, lora_root):
        return getattr(args, "lora_dir_tts", None) or os.path.join(lora_root, "tts")

    @staticmethod
    def query_model_def(base_model_type, model_def):
        return _get_yue_model_def(model_def)

    @staticmethod
    def query_model_files(computeList, base_model_type, model_def=None):
        return _get_yue_download_def(model_def or {})

    @staticmethod
    def load_model(
        model_filename,
        model_type,
        base_model_type,
        model_def,
        quantizeTransformer=False,
        text_encoder_quantization=None,
        dtype=None,
        VAE_dtype=None,
        mixed_precision_transformer=False,
        save_quantized=False,
        submodel_no_list=None,
        text_encoder_filename=None,
        profile=0,
        **kwargs,
    ):
        from .yue.pipeline import YuePipeline

        if isinstance(model_filename, list):
            stage1_weights = model_filename[0] if len(model_filename) > 0 else ""
            stage2_weights = model_filename[1] if len(model_filename) > 1 else ""
        else:
            stage1_weights = model_filename or ""
            stage2_weights = ""

        pipeline = YuePipeline(
            stage1_weights_path=stage1_weights,
            stage2_weights_path=stage2_weights,
            use_audio_prompt=bool(model_def.get("yue_audio_prompt", False)),
            max_new_tokens=model_def.get("yue_max_new_tokens", 200),
            run_n_segments=model_def.get("yue_run_n_segments", 1),
            stage2_batch_size=model_def.get("yue_stage2_batch_size", 10),
            segment_duration=model_def.get("yue_segment_duration", 6),
            prompt_start_time=model_def.get("yue_prompt_start_time", 0.0),
            prompt_end_time=model_def.get("yue_prompt_end_time", 30.0),
        )

        pipe = {
            "transformer": pipeline.model_stage1,
            "transformer2": pipeline.model_stage2,
            "codec_model": pipeline.codec_model,
            "vocoder_vocal": pipeline.vocoder_vocal,
            "vocoder_inst": pipeline.vocoder_inst,
        }
        return pipeline, pipe

    @staticmethod
    def fix_settings(base_model_type, settings_version, model_def, ui_defaults):
        if "alt_prompt" not in ui_defaults:
            ui_defaults["alt_prompt"] = ""

        defaults = {
            "audio_prompt_type": "",
        }
        for key, value in defaults.items():
            ui_defaults.setdefault(key, value)

    @staticmethod
    def update_default_settings(base_model_type, model_def, ui_defaults):
        ui_defaults.update(
            {
                "audio_prompt_type": "",
                "alt_prompt": "pop, dreamy, warm vocal, female, nostalgic",
                "repeat_generation": 1,
                "video_length": 0,
                "num_inference_steps": 0,
                "negative_prompt": "",
                "temperature": 1.0,
                "multi_prompts_gen_type": 2,
            }
        )

    @staticmethod
    def validate_generative_prompt(base_model_type, model_def, inputs, one_prompt):
        if one_prompt is None or len(str(one_prompt).strip()) == 0:
            return "Lyrics prompt cannot be empty for Yue."
        alt_prompt = inputs.get("alt_prompt", "")
        if alt_prompt is None or len(str(alt_prompt).strip()) == 0:
            return "Genres prompt cannot be empty for Yue."
        audio_prompt_type = inputs.get("audio_prompt_type", "") or ""
        if model_def.get("yue_audio_prompt", False):
            if "A" in audio_prompt_type:
                if inputs.get("audio_guide") is None:
                    return "You must provide a vocal or mixed audio prompt for Yue ICL."
                if "B" in audio_prompt_type and inputs.get("audio_guide2") is None:
                    return "You must provide an instrumental prompt for Yue ICL."
                start_time = float(
                    inputs.get(
                        "yue_prompt_start_time",
                        model_def.get("yue_prompt_start_time", 0.0),
                    )
                )
                end_time = float(
                    inputs.get(
                        "yue_prompt_end_time",
                        model_def.get("yue_prompt_end_time", 30.0),
                    )
                )
                if start_time >= end_time:
                    return "Audio prompt start time must be less than end time."
                if end_time - start_time > 30:
                    return "Audio prompt duration should not exceed 30 seconds."
            elif inputs.get("audio_guide") is not None or inputs.get("audio_guide2") is not None:
                return "Select an audio prompt type for Yue ICL or clear audio prompts."
        else:
            if inputs.get("audio_guide") is not None or inputs.get("audio_guide2") is not None:
                return "Yue base model does not support audio prompts. Please use Yue ICL."
        return None