Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

21.7 KB
Raw Blame
# Copyright (c) 2025 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from abc import ABC, abstractmethod
from contextlib import contextmanager
from dataclasses import dataclass

import torch
from tqdm import tqdm

from auto_round.modeling.fused_moe.utils import is_linearized_layout
from auto_round.utils import (
    LazyImport,
    dump_mem_usage,
    dump_memory_usage_ctx,
    global_state,
    is_transformers_version_greater_or_equal_5,
    logger,
)

BUILTIN_MODULES = {
    # Llama4 has no use_experts_implementation, needs custom replacement to handle fused MoE blocks.
    "llama4": LazyImport("auto_round.modeling.fused_moe.llama4"),
    # DeepseekV2Attention enables q_scale calibration for deepseek v2 on Gaudi (#1299)
    "deepseek_v2": LazyImport("auto_round.modeling.fused_moe.deepseek_v2"),
    # Qwen3.5 MoE uses block-local materialization to avoid eagerly unfusing the full model.
    "qwen3_5_moe": LazyImport("auto_round.modeling.fused_moe.qwen3_5_moe"),
    "qwen3_5_moe_text": LazyImport("auto_round.modeling.fused_moe.qwen3_5_moe"),
    # Step 3.5 MoE: splits fused MoELinear into per-expert nn.Linear
    "step3p5": LazyImport("auto_round.modeling.fused_moe.step3_5_moe"),
    # Qwen3-Omni MoE: thinker (no shared expert)
    "qwen3_omni_moe": LazyImport("auto_round.modeling.fused_moe.qwen3_omni"),
    # Kimi remote MoE: routed experts run under @torch.no_grad() in eval mode
    "kimi_vl": LazyImport("auto_round.modeling.fused_moe.kimi"),
    "kimi_linear": LazyImport("auto_round.modeling.fused_moe.kimi"),
    "kimi_k3": LazyImport("auto_round.modeling.fused_moe.kimi"),
}

_FUSED_MOE_PROJECTION_NAMES = frozenset({"gate_up_proj", "gate_proj", "up_proj", "down_proj", "w1", "w2", "w3"})


def _is_fused_moe_parameter(name: str, parameter: torch.Tensor) -> bool:
    return parameter.ndim == 3 and any(part in _FUSED_MOE_PROJECTION_NAMES for part in name.split("."))


# Module name prefixes to exclude from MoE unfusing, keyed by model_type.
# Modules under these prefixes stay in their original fused 3D format during
# quantization. ShardWriter expands them to per-expert 2D tensors at save time.
MOE_SKIP_PREFIXES: dict[str, list[str]] = {
    "qwen3_omni_moe": ["talker."],
}


# transformers >= 5.0.0 supports the general linear_loop experts interface.
if not is_transformers_version_greater_or_equal_5():
    # transformers == 5.2.0: IndexError: Dimension out of range (expected to be in range of [-1, 0], but got 1)
    BUILTIN_MODULES["qwen3_vl_moe"] = LazyImport("auto_round.modeling.fused_moe.qwen3_vl_moe")
    # transformers == 5.2.0: IndexError: index 4 is out of bounds for dimension 0 with size 4
    BUILTIN_MODULES["gpt_oss"] = LazyImport("auto_round.modeling.fused_moe.gpt_oss")


def _handle_moe_modules(model: torch.nn.Module) -> list[str]:
    """Handle fused MOE modules using transformers' linear_loop backend.

    Modules under prefixes listed in MOE_SKIP_PREFIXES for the model's
    model_type are left in their original fused format.

    Args:
        model: The model to process

    Returns:
        List of module names that were processed
    """
    from auto_round.modeling.fused_moe.moe_experts_interface import (
        is_linear_loop_available,
        prepare_model_for_moe_quantization,
    )

    if not is_linear_loop_available():
        logger.warning(
            "transformers' linear_loop experts interface not available (requires transformers 5.0+). "
            "MOE modules with @use_experts_implementation decorator will fall back to custom replacements "
            "if registered."
        )
        return []

    model_type = getattr(getattr(model, "config", None), "model_type", None)
    skip_prefixes = MOE_SKIP_PREFIXES.get(model_type, [])

    # Use transformers' experts interface
    unfused = prepare_model_for_moe_quantization(model, skip_prefixes=skip_prefixes)
    if unfused:
        logger.info(f"Prepared {len(unfused)} MOE modules for quantization")
    return unfused


def _import_required_replacements(model: torch.nn.Module) -> None:
    """Scan model and trigger lazy imports for registered replacement modules."""
    if not is_custom_model(model):
        return
    model_type = model.config.model_type
    _ = BUILTIN_MODULES[model_type].__name__  # Trigger lazy import
    logger.debug(f"Loaded replacement module for {model_type}")


def is_custom_model(model: torch.nn.Module) -> bool:
    """Check if the model has a custom replacement registered via BUILTIN_MODULES.

    Returns True if the model's model_type matches a key in BUILTIN_MODULES.
    """
    if hasattr(model, "config") and hasattr(model.config, "model_type"):
        return model.config.model_type in BUILTIN_MODULES
    return False


def _find_first_moe_block(model: torch.nn.Module) -> tuple[str, torch.nn.Module] | tuple[None, None]:
    """Return ``(name, module)`` of the first experts-like module, or ``(None, None)``."""
    for name, module in model.named_modules():
        if name.endswith((".experts", ".moe")):
            return name, module
    return None, None


def _describe_moe_block(model: torch.nn.Module, name: str) -> str | None:
    """Render the experts module at ``name`` for the before/after comparison."""
    try:
        module = model.get_submodule(name)
    except AttributeError:
        return None
    return f"({module.__class__.__name__}):\n{module}"


@contextmanager
def log_moe_block_transition(model: torch.nn.Module, label: str = "replacement"):
    """Log the first experts module before and after a structural change.

    The structural unfuse happens in one of two places depending on how the model was
    built: on the meta skeleton (``build_meta_model``) or during ``apply_replacements``.
    Both wrap themselves in this so the transition is reported wherever it actually
    occurs, and only when something really changed -- printing an identical before/after
    pair (which is what the other path sees, since it then has nothing left to do) is
    just noise.
    """
    name, _ = _find_first_moe_block(model)
    before = _describe_moe_block(model, name) if name is not None else None
    try:
        yield
    finally:
        if name is not None:
            after = _describe_moe_block(model, name)
            if after is not None and after != before:
                logger.info(f"Experts (before {label}) [{name}] {before}")
                logger.info(f"Experts (after {label}) [{name}] {after}")
            else:
                logger.debug(f"Experts [{name}] left unchanged by {label} {before}")


@dump_mem_usage("Materializing model", log_level="debug")
def materialize_model_(model: torch.nn.Module) -> None:
    def _materialize_module(module: torch.nn.Module) -> None:
        if isinstance(module, ReplacementModuleBase):
            module.materialize_weights()

    model.apply(_materialize_module)

    # Any tensor still on meta after per-block materialization means the meta-skeleton
    # loader could not resolve it from the checkpoint (e.g. a checkpoint-conversion
    # layout AutoRound does not reverse yet). Leaving it on meta only defers the failure
    # to a cryptic "Cannot copy out of meta tensor" crash on the next `.to(device)`, so
    # fail fast with an actionable message.
    still_meta = [name for name, param in model.named_parameters() if param.device.type == "meta"]
    still_meta += [name for name, buffer in model.named_buffers() if buffer.device.type == "meta"]
    if still_meta:
        from auto_round import envs

        preview = ", ".join(still_meta[:8]) + (f" (+{len(still_meta) - 8} more)" if len(still_meta) > 8 else "")
        # The auto meta-skeleton is only enabled for fused-MoE checkpoints; an explicit
        # AR_DISK_STREAM_MODEL=1 forces it for anything else. Point at whichever toggle
        # actually turned it on so the hint stays correct (AR_DISABLE_META_LOAD is
        # the MoE-only auto path).
        hint = "unset AR_DISK_STREAM_MODEL" if envs.AR_DISK_STREAM_MODEL else "set AR_DISABLE_META_LOAD=1"
        raise RuntimeError(
            f"Failed to materialize {len(still_meta)} checkpoint tensor(s), still on meta: {preview}. "
            f"To load the whole model on CPU instead, {hint} (uses more RAM)."
        )
    logger.debug("All parameters and buffers have been materialized from meta device.")
    release_original_module_(model)


def release_original_module_(model: torch.nn.Module) -> None:
    def _clear_source_module(module: torch.nn.Module) -> None:
        if isinstance(module, ReplacementModuleBase):
            module.release_original_module()

    model.apply(_clear_source_module)


def safe_to_cpu_(model: torch.nn.Module) -> None:
    """Move a model to CPU unless it still contains unmaterialized meta tensors."""
    has_meta_tensor = any(param.device.type == "meta" for param in model.parameters()) or any(
        buffer.device.type == "meta" for buffer in model.buffers()
    )
    if not has_meta_tensor:
        model.to("cpu")


class ReplacementModuleBase(ABC, torch.nn.Module):
    """
    Abstract base class for module replacement during calibration phase.

    Replacement modules replace original modules to ensure all components
    receive data for proper quantization statistics.

    Subclasses must:
    1. Implement `original_module_class()` to return the target module class name
    2. Implement `__init__()` with signature:
       (self, original, config)
    """

    # Registry: module class name -> replacement module class
    _replacement_registry: dict[str, type["ReplacementModuleBase"]] = {}
    supports_gguf_fused_moe: bool = False
    # Set False for replacements that only patch behavior and also apply to linearized experts.
    skip_linearized_experts: bool = True

    def __init_subclass__(cls, **kwargs):
        """Automatically register subclasses in the replacement registry."""
        super().__init_subclass__(**kwargs)

        # Only register if it's a concrete implementation (not ABC)
        if not getattr(cls, "__abstractmethods__", None):
            if cls.original_module_class() is None:
                raise TypeError(
                    f"{cls.__name__} must implement 'original_module_class()' class method "
                    "to return the name of the module class it replaces"
                )

            if cls.original_module_class() in cls._replacement_registry:
                existing = cls._replacement_registry[cls.original_module_class()]
                raise ValueError(
                    f"Module '{cls.original_module_class()}' already registered to "
                    f"{existing.__name__}. Cannot register {cls.__name__}."
                )

            cls._replacement_registry[cls.original_module_class()] = cls
            logger.trace(f"Registered {cls.__name__} for replacing {cls.original_module_class()}")

    def __init__(self, original: torch.nn.Module):
        super().__init__()
        if any(_is_fused_moe_parameter(name, parameter) for name, parameter in original.named_parameters()):
            self._auto_round_replaced_fused_moe = True
        _global_tracker.register_replacement(
            name=str(id(self)),
            original=original,
            replacement=self,
        )
        self._materialized = False

    @classmethod
    def get_replacement_class(cls, module_class_name: str) -> type["ReplacementModuleBase"]:
        """Get replacement class for a given module class name."""
        return cls._replacement_registry.get(module_class_name)

    @classmethod
    def is_registered(cls, module_class_name: str) -> bool:
        """Check if a module class has a replacement implementation."""
        return module_class_name in cls._replacement_registry

    @classmethod
    def is_to_be_replaced(
        cls,
        original: torch.nn.Module,
    ) -> bool:
        """Determine if the given module should be replaced.

        Users can extend this method to add custom logic for replacement.
        """
        return cls.is_registered(original.__class__.__name__)

    @classmethod
    def get_registered_modules(cls) -> list:
        """Get list of all registered module class names."""
        return list(cls._replacement_registry.keys())

    @classmethod
    @abstractmethod
    def original_module_class(cls) -> str:
        """Return the class name of the module this replaces."""

    @classmethod
    @abstractmethod
    def from_original(
        cls,
        original: torch.nn.Module,
        config,
    ) -> "ReplacementModuleBase":
        """Create replacement module from original module."""

    def materialize_weights(self):
        """Materialize weights if needed."""
        if not self._materialized:
            self._materialize_weights()
            self.post_process_materialization()

    def _materialize_weights(self) -> None:
        """Materialize weights from the original module.

        Subclasses should override this method to implement
        weight materialization logic.
        """

    def release_original_module(self) -> None:
        """Release reference to the original module to free memory."""
        # Release from global tracker
        _global_tracker.release_original(self)

    def _get_original_module(self) -> torch.nn.Module:
        """Get the original module associated with this replacement."""
        return _global_tracker.get_original(self)

    def post_process_materialization(self) -> None:
        """Mark the replacement module as materialized."""
        self._materialized = True
        self.release_original_module()


# Note: adapted from llm-compressor
# https://github.com/vllm-project/llm-compressor/blob/main/src/llmcompressor/modeling/moe_context.py


def apply_replacements(
    model: torch.nn.Module,
    auto_detect_moe: bool = True,
    gguf_export: bool = False,
) -> torch.nn.Module:
    """
    Function to apply module replacements to a model.

    This scans all modules in the model and replaces any registered modules with their
    replacement equivalents. Non-permanent modules are tracked for later restoration.

    The model is modified in-place, so the same model object should be used.

    Args:
        model: The model to apply module replacement to (modified in-place).
        auto_detect_moe: If True, automatically detect and handle fused MOE modules
            (transformers 5.0+ pattern). Default is True.
        gguf_export: If True, skip custom fused-MoE replacements without an explicit
            reversible GGUF fusion specification.

    Returns:
        The model with modules replaced.
    """
    _import_required_replacements(model)

    with log_moe_block_transition(model, "replacement"):
        # Custom replacements first
        if is_custom_model(model):
            _apply_custom_replacements(model, gguf_export=gguf_export)

        if auto_detect_moe and is_transformers_version_greater_or_equal_5():
            _handle_moe_modules(model)

    return model


@dump_mem_usage("applying custom replacements")
def _apply_custom_replacements(model: torch.nn.Module, gguf_export: bool = False) -> list:
    """Scan model and replace registered modules with custom implementations.

    Args:
        model: The model to scan and apply replacements to (modified in-place).

    Returns:
        List of (name, replacement_class) tuples for replaced modules.
    """
    replaced = []

    # Step 1: Collect all modules that need replacement
    logger.debug("Scanning for modules to replace")
    modules_to_replace = []
    skipped_modules = []
    for name, module in model.named_modules():
        # skip replaced modules
        if isinstance(module, ReplacementModuleBase):
            continue
        class_name = module.__class__.__name__
        if ReplacementModuleBase.is_registered(class_name):
            replacement_cls = ReplacementModuleBase.get_replacement_class(class_name)
            if (
                gguf_export
                and any(_is_fused_moe_parameter(name, parameter) for name, parameter in module.named_parameters())
                and not replacement_cls.supports_gguf_fused_moe
            ):
                skipped_modules.append((name, class_name, "skipped because GGUF fusion spec is unavailable"))
                continue
            # If llm-compressor already linearized experts, skip custom replacement.
            if (
                replacement_cls.skip_linearized_experts
                and hasattr(module, "experts")
                and is_linearized_layout(module.experts)
            ):
                skipped_modules.append((name, class_name, "skipped because linearize_moe"))
                continue
            if replacement_cls.is_to_be_replaced(module):
                modules_to_replace.append((name, module, class_name))
            else:
                skipped_modules.append((name, class_name, "skipped because replacement criteria not met"))

    if skipped_modules:
        for name, class_name, reason in skipped_modules:
            logger.info(f"Skipped replacement for {name} ({class_name}): {reason}")

    # Step 2: Replace modules
    if modules_to_replace:
        logger.info(f"Found {len(modules_to_replace)} modules to replace")
        for name, module, class_name in tqdm(modules_to_replace, desc="Replacing modules"):
            module = model.get_submodule(name)
            # The module might have been replaced earlier in the loop (parent-first replacement).
            # Skip if the class has changed or it no longer matches replacement criteria.
            if module.__class__.__name__ != class_name:
                logger.debug(
                    f"Skipping replacement for {name}: class changed from {class_name} to {module.__class__.__name__}"
                )
                continue
            with dump_memory_usage_ctx(f"Replacing module {name}", log_level="debug"):
                replacement_cls = ReplacementModuleBase.get_replacement_class(class_name)
                if not replacement_cls.is_to_be_replaced(module):
                    logger.debug(f"Skipping replacement for {name}: no longer matches replacement criteria")
                    continue
                replacement = replacement_cls.from_original(
                    module,
                    model.config,
                )
                model.set_submodule(name, replacement)
                replaced.append((name, replacement_cls))
    else:
        logger.debug("No modules found for replacement")

    # Log what was replaced
    if replaced:
        global_state.replaced_module_count = len(replaced)
        logger.info(f"Replaced {len(replaced)} modules")

    return replaced


@dataclass
class ReplacedModuleInfo:
    original_module: torch.nn.Module
    replacement_module: ReplacementModuleBase


class ModuleReplacementTracker:
    """Tracker to maintain mapping between replacement modules and their original modules.

    This is a singleton class - only one instance can exist.
    """

    _instance = None
    _initialized = False

    def __new__(cls):
        if cls._instance is None:
            cls._instance = super().__new__(cls)
        return cls._instance

    def __init__(self):
        # Only initialize once
        if ModuleReplacementTracker._initialized:
            return

        # Map from replacement module id to original module
        self._replacement_to_original: dict[int, torch.nn.Module] = {}
        # Map from module name to ReplacedModuleInfo
        self._name_to_info: dict[str, ReplacedModuleInfo] = {}

        ModuleReplacementTracker._initialized = True

    @classmethod
    def get_instance(cls) -> "ModuleReplacementTracker":
        """Get the singleton instance of the tracker."""
        if cls._instance is None:
            cls._instance = cls()
        return cls._instance

    def register_replacement(self, name: str, original: torch.nn.Module, replacement: ReplacementModuleBase) -> None:
        """Register a module replacement."""
        self._replacement_to_original[id(replacement)] = original
        self._name_to_info[name] = ReplacedModuleInfo(original_module=original, replacement_module=replacement)
        logger.trace(f"Registered replacement for module: {name}")

    def get_original(self, replacement: ReplacementModuleBase) -> torch.nn.Module:
        """Get the original module for a given replacement module."""
        return self._replacement_to_original.get(id(replacement))

    def get_info_by_name(self, name: str) -> ReplacedModuleInfo:
        """Get replacement info by module name."""
        return self._name_to_info.get(name)

    def release_original(self, replacement: ReplacementModuleBase) -> None:
        """Release the original module associated with a replacement module."""
        replacement_id = id(replacement)
        if replacement_id in self._replacement_to_original:
            original = self._replacement_to_original[replacement_id]
            # Delete the original module to free memory
            del original
            del self._replacement_to_original[replacement_id]
            logger.trace(f"Released original module for replacement {replacement_id}")

    def release_all_originals(self) -> None:
        """Release all tracked original modules."""
        count = len(self._replacement_to_original)
        if count > 0:
            self._replacement_to_original.clear()
            logger.debug(f"Released {count} original modules from tracker")

    def clear(self) -> None:
        """Clear all tracked information."""
        self._replacement_to_original.clear()
        self._name_to_info.clear()
        logger.debug("Cleared module replacement tracker")


_global_tracker = ModuleReplacementTracker()