Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

24.0 KB
Raw Blame
import concurrent.futures
import copy
import os
import re
import shutil

import pytest
import torch
import transformers
from packaging import version

from auto_round.eval.evaluation import simple_evaluate, simple_evaluate_user_model
from auto_round.utils import diffusion_load_model, get_attr, get_major_device, llm_load_model, mllm_load_model, set_attr

transformers_version = version.parse(transformers.__version__)


def _raise_threaded_packing(*args, **kwargs):
    raise AssertionError("Packing should not create a thread pool or call threadpoolctl.")


def forbid_threaded_packing(monkeypatch, module):
    monkeypatch.setattr(concurrent.futures, "ThreadPoolExecutor", _raise_threaded_packing)
    monkeypatch.setattr(module, "ThreadPoolExecutor", _raise_threaded_packing, raising=False)
    tctl = getattr(module, "tctl", None)
    if tctl is not None:
        monkeypatch.setattr(tctl, "threadpool_limits", _raise_threaded_packing)


def generate_prompt(
    model_obj_or_str,
    tokenizer=None,
    text="The capital of France is,",
    max_new_tokens=10,
    device=None,
    include_prompt=True,
):
    """Generate text using a model and tokenizer.

    Args:
        model_obj_or_str: The model to use for generation.
        tokenizer: The tokenizer for the model.
        text: The input prompt text.
        max_new_tokens: Maximum number of new tokens to generate.

    Returns:
        str: The generated text. If include_prompt is False, returns only
            newly generated continuation tokens.
    """
    if device is None:
        device = get_major_device()
    if isinstance(model_obj_or_str, str):
        model, tokenizer = llm_load_model(model_obj_or_str, trust_remote_code=True)
    else:
        model = model_obj_or_str
        assert tokenizer is not None, "Tokenizer must be provided when model is a model object"
    if not (hasattr(model, "hf_device_map") and len(model.hf_device_map) > 1):
        model = model.to(device)
    inputs = tokenizer(text, return_tensors="pt").to(model.device)
    generated_ids = model.generate(**inputs, max_new_tokens=max_new_tokens, do_sample=False)[0]
    if include_prompt:
        output_ids = generated_ids
    else:
        prompt_len = inputs["input_ids"].shape[-1]
        output_ids = generated_ids[prompt_len:]
    output = tokenizer.decode(output_ids)
    print(output)
    return output


def eval_generated_prompt(
    model,
    tokenizer=None,
    prompt_text="United States of",
    target_text="America",
    max_new_tokens=10,
    device=None,
):
    # Use a completion-style prompt: base models (e.g. OPT-125M) cannot follow
    # instructions, so an instruction prompt makes the check flaky after quantization.
    generated_text = generate_prompt(
        model,
        tokenizer,
        prompt_text,
        max_new_tokens=max_new_tokens,
        device=device,
        include_prompt=False,
    )
    # accept lowercase target text as well, since some models may not follow the prompt exactly
    assert target_text.lower() in generated_text.lower(), f"Expected {target_text} in generated text: {generated_text}"


def evaluate_accuracy(
    model_or_save_dir,
    tokenizer=None,
    task="lambada_openai",
    threshold=0.25,
    batch_size="auto",
    limit=None,
    device=None,
    model_type="hf",
):
    """Helper function to evaluate model accuracy on a given task.

    Supports both saved model directory and in-memory model object.

    Args:
        model_or_save_dir: Either a path to the saved model directory (str) or a model object.
        tokenizer: The tokenizer for the model (required when model_or_save_dir is a model object).
        task: The evaluation task.
        threshold: The minimum accuracy threshold.
        batch_size: Batch size for evaluation.
        limit: Limit the number of samples to evaluate (only for model object).

    Returns:
        float: The accuracy value.

    Raises:
        AssertionError: If accuracy is below threshold.
    """
    if isinstance(model_or_save_dir, str):
        # save_dir mode
        model_args = f"pretrained={model_or_save_dir}"
        result = simple_evaluate(
            model=model_type, model_args=model_args, tasks=task, batch_size=batch_size, limit=limit, device=device
        )
    else:
        # model object mode
        if tokenizer is None:
            raise ValueError("tokenizer is required when model_or_save_dir is a model object")
        if device and device != "cpu":
            model_or_save_dir = model_or_save_dir.to(device)
        result = simple_evaluate_user_model(
            model_or_save_dir, tokenizer, batch_size=batch_size, tasks=task, limit=limit, device=device
        )

    acc = result["results"][task]["acc,none"]
    print(f"{task} accuracy: {acc}")
    assert acc > threshold, f"Accuracy {acc} is below threshold {threshold}"
    return acc


# Automatic choose local path or model name.
def get_model_path(model_name: str) -> str:
    model_name = model_name.rstrip("/")
    ut_path = f"/tf_dataset/auto_round/models/{model_name}"
    local_path = f"/models/{model_name.split('/')[-1]}"
    local_path_1 = f"/dataset/{model_name.split('/')[-1]}"

    if os.path.exists(ut_path):
        return ut_path
    elif os.path.exists(local_path):
        return local_path
    elif os.path.exists(local_path_1):
        return local_path_1
    else:
        return model_name


opt_name_or_path = get_model_path("facebook/opt-125m")
qwen_name_or_path = get_model_path("Qwen/Qwen3-0.6B")
lamini_name_or_path = get_model_path("MBZUAI/LaMini-GPT-124M")
gptj_name_or_path = get_model_path("hf-internal-testing/tiny-random-GPTJForCausalLM")
phi2_name_or_path = get_model_path("microsoft/phi-2")
deepseek_v2_name_or_path = get_model_path("deepseek-ai/DeepSeek-V2-Lite")
qwen_moe_name_or_path = get_model_path("Qwen/Qwen1.5-MoE-A2.7B")
qwen_vl_name_or_path = get_model_path("Qwen/Qwen2-VL-2B-Instruct")
qwen_2_5_vl_name_or_path = get_model_path("Qwen/Qwen2.5-VL-3B-Instruct")
qwen_3_vl_9b_name_or_path = get_model_path("Qwen/Qwen3.5-9B")
gemma_name_or_path = get_model_path("benzart/gemma-2b-it-fine-tuning-for-code-test")
qwen2_5_omni_name_or_path = get_model_path("Qwen/Qwen2.5-Omni-3B")
qwen3_omni_name_or_path = get_model_path("Qwen/Qwen3-Omni-30B-A3B-Instruct")
flux_name_or_path = get_model_path("black-forest-labs/FLUX.1-dev")


def make_tiny_qwen3_omni_moe_config():
    """Build a self-contained Qwen3-Omni-MoE config for unit-test smoke coverage."""
    config = transformers.Qwen3OmniMoeConfig()

    thinker = config.thinker_config
    thinker_text = thinker.text_config
    thinker_text.num_hidden_layers = 1
    thinker_text.hidden_size = 64
    thinker_text.intermediate_size = 128
    thinker_text.moe_intermediate_size = 32
    thinker_text.num_attention_heads = 4
    thinker_text.num_key_value_heads = 2
    thinker_text.num_experts = 4
    thinker_text.num_experts_per_tok = 2
    # The text-only forward still reads these multimodal sentinel ids when building positions.
    thinker.audio_token_id = 151646
    thinker.image_token_id = 151655
    thinker.video_token_id = 151656
    thinker.vision_start_token_id = 151652
    thinker.audio_start_token_id = 151647

    vision = thinker.vision_config
    vision.depth = 1
    vision.hidden_size = 64
    vision.num_heads = 4

    audio = thinker.audio_config
    audio.num_hidden_layers = 1
    audio.d_model = 64
    audio.encoder_attention_heads = 4
    audio.encoder_ffn_dim = 128
    audio.output_dim = 64
    audio.downsample_hidden_size = 32
    audio.num_mel_bins = 16

    talker = config.talker_config
    talker_text = talker.text_config
    talker_text.num_hidden_layers = 1
    talker_text.hidden_size = 64
    talker_text.intermediate_size = 128
    talker_text.moe_intermediate_size = 32
    talker_text.num_attention_heads = 4
    talker_text.num_key_value_heads = 2
    talker_text.num_experts = 4
    talker_text.num_local_experts = 4
    talker_text.num_experts_per_tok = 2
    talker_text.shared_expert_intermediate_size = 64

    predictor = talker.code_predictor_config
    predictor.vocab_size = 64
    predictor.hidden_size = 64
    predictor.intermediate_size = 128
    predictor.num_hidden_layers = 1
    predictor.num_attention_heads = 4
    predictor.num_key_value_heads = 2
    predictor.head_dim = 16
    predictor.num_code_groups = 2
    predictor.layer_types = ["full_attention"]
    talker.num_code_groups = 2
    talker.thinker_hidden_size = 64
    talker.spatial_merge_size = 2

    code2wav = config.code2wav_config
    code2wav.codebook_size = 64
    code2wav.hidden_size = 64
    code2wav.num_hidden_layers = 1
    code2wav.num_attention_heads = 4
    code2wav.num_key_value_heads = 4
    code2wav.intermediate_size = 128
    code2wav.num_quantizers = 2
    code2wav.decoder_dim = 64
    code2wav.upsample_rates = [2]
    code2wav.upsampling_ratios = [2]

    config.initializer_range = 0.02
    return config


# Slice model into tiny model for speedup
def _reduce_config_layers(config, num_layers, num_experts=None):
    """Reduce num_layers and num_experts in a config object (in-place).

    Handles nested sub-configs (text_config, vision_config, audio_config,
    thinker_config, talker_config, etc.) commonly found in multi-modal and
    MoE architectures.
    """
    n_block_keys = [
        "n_layers",
        "num_hidden_layers",
        "n_layer",
        "num_layers",
        "depth",
        "encoder_layers",
        "num_single_layers",
        "num_decoder_layers",
        "input_local_layers",
        "local_layers",
    ]
    n_expert_keys = ["num_experts", "num_local_experts"]

    def _apply(cfg):
        for key in n_block_keys:
            if isinstance(cfg, dict) and key in cfg:
                cfg[key] = num_layers
            elif hasattr(cfg, key):
                setattr(cfg, key, num_layers)

        if isinstance(cfg, dict) and "layer_types" in cfg:
            cfg["layer_types"] = cfg["layer_types"][:num_layers]
        elif hasattr(cfg, "layer_types"):
            cfg.layer_types = cfg.layer_types[:num_layers]

        if num_experts is not None:
            for key in n_expert_keys:
                if isinstance(cfg, dict) and key in cfg:
                    cfg[key] = num_experts
                elif hasattr(cfg, key):
                    setattr(cfg, key, num_experts)

    # Top-level config
    _apply(config)

    # Common sub-configs
    for sub_name in [
        "text_config",
        "vision_config",
        "audio_config",
        "thinker_config",
        "talker_config",
        "code_predictor_config",
    ]:
        sub_cfg = getattr(config, sub_name, None)
        if sub_cfg is not None:
            _apply(sub_cfg)
            # Handle deeper nesting, e.g. thinker_config.text_config
            for inner_name in ["text_config", "vision_config", "audio_config"]:
                inner_cfg = getattr(sub_cfg, inner_name, None)
                if inner_cfg is not None:
                    _apply(inner_cfg)


def _apply_config_overrides(config, overrides):
    """Apply arbitrary key-value overrides to a config (dict or object)."""
    if not overrides:
        return
    for key, value in overrides.items():
        if isinstance(config, dict):
            if key in config:
                config[key] = value
        elif hasattr(config, key):
            setattr(config, key, value)


def get_tiny_model(
    model_name_or_path,
    num_layers=2,
    num_experts=None,
    is_mllm=False,
    from_config=False,
    is_diffusion=False,
    config_overrides=None,
    **kwargs,
):
    """Generate a tiny model by slicing layers from the original model.

    Args:
        model_name_or_path: HuggingFace model name or local path.
        num_layers: Number of layers to keep.
        num_experts: Number of experts to keep for MoE models (None = unchanged).
        is_mllm: Whether the model is a multi-modal model.
        from_config: If True, initialise the model from config with random
            weights instead of downloading the full checkpoint.  This is much
            faster and avoids large downloads.
        **kwargs: Extra keyword arguments forwarded to the model loader or
            ``AutoConfig.from_pretrained``.
    """
    if "use_config" in kwargs:
        from_config = kwargs.pop("use_config")

    model_name_or_path = get_model_path(model_name_or_path)

    if from_config:
        # ---- lightweight path: config-only, random weights ----
        if is_diffusion:
            import importlib
            import json

            from diffusers import AutoPipelineForText2Image
            from huggingface_hub import snapshot_download

            diffusers_module = importlib.import_module("diffusers")
            transformers_module = importlib.import_module("transformers")

            existing = False
            if os.path.exists(model_name_or_path):
                local_dir = model_name_or_path
                existing = True
            else:
                local_dir = snapshot_download(
                    repo_id=model_name_or_path, ignore_patterns=["*.safetensors", "*.safetensors.index.json"]
                )

            def _get_module(cls_name, mod_name, folder_name):
                if cls_name == "diffusers":
                    with open(os.path.join(local_dir, folder_name, "config.json"), "r", encoding="utf-8") as f:
                        config = json.load(f)
                    _reduce_config_layers(config, num_layers, num_experts)
                    _apply_config_overrides(config, config_overrides)
                    return getattr(diffusers_module, mod_name).from_config(config)
                else:
                    config = transformers.AutoConfig.from_pretrained(
                        os.path.join(local_dir, folder_name, "config.json")
                    )
                    _reduce_config_layers(config, num_layers, num_experts)
                    _apply_config_overrides(config, config_overrides)
                    return getattr(transformers_module, mod_name)._from_config(config)

            with open(os.path.join(local_dir, "model_index.json"), "r", encoding="utf-8") as f:
                model_index = json.load(f)

            if not existing:
                for k, v in model_index.items():
                    if k in ["scheduler", "tokenizer", "tokenizer_2"]:
                        continue
                    if isinstance(v, list) and v[0] in ["diffusers", "transformers"]:
                        module = _get_module(v[0], v[1], k)
                        module.save_pretrained(os.path.join(local_dir, k))
                model = AutoPipelineForText2Image.from_pretrained(local_dir)
            else:
                model = AutoPipelineForText2Image.from_pretrained(local_dir)
                for k, v in model_index.items():
                    if (
                        k not in ["scheduler", "tokenizer", "tokenizer_2"]
                        and isinstance(v, list)
                        and v[0] in ["diffusers", "transformers"]
                    ):
                        tiny_module = _get_module(v[0], v[1], k)
                        setattr(model, k, tiny_module)
            return model
        else:
            trust_remote_code = kwargs.get("trust_remote_code", True)
            config = transformers.AutoConfig.from_pretrained(model_name_or_path, trust_remote_code=trust_remote_code)
            # Special cases, for transformers == 5.4.0
            if config.model_type == "qwen3_omni_moe":
                config.initializer_range = 0.02  # Default initializer range for weight initialization
            _reduce_config_layers(config, num_layers, num_experts)
            _apply_config_overrides(config, config_overrides)

            # Pick the right model class
            base_lib = transformers
            architectures = getattr(config, "architectures", [None])[0]
            if (
                is_mllm
                and architectures.endswith("Model")
                and hasattr(base_lib, n := architectures.replace("Model", "ForConditionalGeneration"))
            ):
                model_cls = getattr(base_lib, n)
            elif hasattr(base_lib, architectures):
                model_cls = getattr(base_lib, architectures)
            else:
                model_cls = transformers.AutoModelForCausalLM  # default to causal LM if we can't find a better match
            model = model_cls._from_config(config)
            model = model.eval()
            return model

    # ---- original path: load pretrained weights then slice ----
    def slice_layers(module):
        """slice layers in the model."""
        sliced = False
        for name, child in module.named_children():
            if isinstance(child, torch.nn.ModuleList) and len(child) > num_layers:
                new_layers = torch.nn.ModuleList(child[:num_layers])
                setattr(module, name, new_layers)
                sliced = True
            elif slice_layers(child):
                sliced = True
        return sliced

    kwargs["dtype"] = "auto" if "auto" not in kwargs else kwargs["dtype"]
    kwargs["trust_remote_code"] = kwargs.get("trust_remote_code", True)
    if is_mllm:
        model, processor, tokenizer, image_processor = mllm_load_model(model_name_or_path, **kwargs)
        if hasattr(model.config, "vision_config"):
            if hasattr(model.config.vision_config, "num_hidden_layers"):  # mistral, etc.
                model.config.num_hidden_layers = num_layers
            elif hasattr(model.config.vision_config, "depth"):  # qwen vl
                model.config.vision_config.depth = num_layers
    elif is_diffusion:
        pipe, model = diffusion_load_model(model_name_or_path, **kwargs)
        if (
            hasattr(model, "config")
            and hasattr(model.config, "num_layers")
            and hasattr(model.config, "num_single_layers")
        ):
            model.config.num_layers = num_layers
            model.config.num_single_layers = num_layers
    else:
        model, tokenizer = llm_load_model(model_name_or_path, **kwargs)

    slice_layers(model)

    _reduce_config_layers(model.config, num_layers, num_experts)

    return model


# for fixture usage only
def save_tiny_model(
    model_name_or_path,
    tiny_model_path,
    num_layers=2,
    num_experts=None,
    is_mllm=False,
    force_untie=False,
    from_config=False,
    is_diffusion=False,
    config_overrides=None,
    **kwargs,
):
    """Generate  a tiny model and save to the specified path.

    Args:
        model_name_or_path: HuggingFace model name or local path.
        tiny_model_path: Where to save the tiny model.
        num_layers: Number of layers to keep.
        num_experts: Number of experts to keep for MoE models (None = unchanged).
        is_mllm: Whether the model is a multi-modal model.
        force_untie: Force untie word embeddings.
        from_config: If True, initialise the model from config with random
            weights instead of downloading the full checkpoint.
        **kwargs: Extra keyword arguments forwarded to the model loader.
    """
    if "use_config" in kwargs:
        from_config = kwargs.pop("use_config")

    model = get_tiny_model(
        model_name_or_path,
        num_layers=num_layers,
        num_experts=num_experts,
        is_mllm=is_mllm,
        from_config=from_config,
        is_diffusion=is_diffusion,
        config_overrides=config_overrides,
        **kwargs,
    )
    if force_untie:
        if getattr(getattr(model, "config", None), "tie_word_embeddings", False):
            model.config.tie_word_embeddings = False
            for key in model._tied_weights_keys:
                weight = get_attr(model, key)
                set_attr(model, key, copy.deepcopy(weight))
    test_path = os.path.dirname(__file__)
    tiny_model_path = os.path.join(test_path, tiny_model_path.removeprefix("./"))
    shutil.rmtree(tiny_model_path, ignore_errors=True)

    if not kwargs.get("trust_remote_code", True) and getattr(getattr(model, "config", None), "auto_map", None):
        del model.config.auto_map

    model.save_pretrained(tiny_model_path)
    if not is_diffusion:
        tokenizer = transformers.AutoTokenizer.from_pretrained(model_name_or_path, **kwargs)
        tokenizer.save_pretrained(tiny_model_path)
    # copy tokenizer.model
    model_path = model_name_or_path
    if not os.path.isdir(model_name_or_path):
        from huggingface_hub.errors import GatedRepoError, HfHubHTTPError, HFValidationError

        from auto_round.utils import download_hf_model

        try:
            model_path = download_hf_model(model_name_or_path)
        except (GatedRepoError, HfHubHTTPError, HFValidationError):
            model_path = model_name_or_path
    if os.path.isfile(os.path.join(model_path, "tokenizer.model")):
        shutil.copy(os.path.join(model_path, "tokenizer.model"), os.path.join(tiny_model_path, "tokenizer.model"))
    if is_mllm:
        sources = [model_path]
        if model_name_or_path not in sources:
            sources.insert(0, model_name_or_path)

        for source in sources:
            try:
                processor = transformers.AutoProcessor.from_pretrained(source, **kwargs)
                processor.save_pretrained(tiny_model_path)
                break
            except (OSError, ValueError):
                continue

        for source in sources:
            try:
                image_processor = transformers.AutoImageProcessor.from_pretrained(source, **kwargs)
                image_processor.save_pretrained(tiny_model_path)
                break
            except (OSError, ValueError):
                continue
    print(f"[Fixture]: built tiny model path:{tiny_model_path} for testing in session")
    return tiny_model_path


# HPU mode checking
def is_pytest_mode_compile():
    return pytest.mode == "compile"


def is_pytest_mode_lazy():
    return pytest.mode == "lazy"


def check_version(lib):
    try:
        from transformers.utils.versions import require_version

        require_version(lib)
        return True
    except Exception:
        return False


# General model inference code
def model_infer(model, tokenizer, apply_chat_template=False):
    """Run model inference and print generated outputs."""
    prompts = [
        "Hello,my name is",
        # "The president of the United States is",
        # "The capital of France is",
        # "The future of AI is",
    ]
    if apply_chat_template:
        texts = []
        for prompt in prompts:
            messages = [{"role": "user", "content": prompt}]
            text = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
            texts.append(text)
        prompts = texts

    inputs = tokenizer(prompts, return_tensors="pt", padding=False, truncation=True)

    outputs = model.generate(
        input_ids=inputs["input_ids"].to(model.device),
        attention_mask=inputs["attention_mask"].to(model.device),
        do_sample=False,  ## change this to follow official usage
        max_new_tokens=5,
    )
    generated_ids = [output_ids[len(input_ids) :] for input_ids, output_ids in zip(inputs["input_ids"], outputs)]

    decoded_outputs = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)

    for i, prompt in enumerate(prompts):
        print(f"Prompt: {prompt}")
        print(f"Generated: {decoded_outputs[i]}")
        print("-" * 50)
    return decoded_outputs[0]


# Dummy dataloader for testing
class DataLoader:

    def __init__(self):
        self.batch_size = 1

    def __iter__(self):
        for i in range(2):
            yield torch.ones([1, 10], dtype=torch.long)


fixed_input = torch.tensor([[10, 20, 30, 40, 50]], dtype=torch.long)


def get_output(model_name_or_path):
    """Get model output for fixed input."""
    try:
        model, tokenizer = llm_load_model(model_name_or_path)
    except:
        model, processor, tokenizer, image_processor = mllm_load_model(model_name_or_path)
    outputs = model(fixed_input)[0]
    return outputs.detach().cpu()


def is_cuda_support_fp8(major=9, minor=0):
    """Check if the current CUDA device capability is >= (major, minor).

    Args:
        major: Required major compute capability (default: 9).
        minor: Required minor compute capability (default: 0).

    Returns:
        bool: True if CUDA is available and device capability >= (major, minor).
    """
    if not torch.cuda.is_available():
        return False
    cap = torch.cuda.get_device_capability()
    return cap >= (major, minor)