Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

29.5 KB
Raw Blame
# Copyright (c) 2026 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import copy
import json
import math
import os
from dataclasses import replace
from typing import Any

import torch

from auto_round.logger import logger
from auto_round.utils import clear_memory
from auto_round.utils.device import (
    dispatch_model_block_wise,
    get_major_device,
)
from auto_round.utils.device_manager import device_manager, is_auto_device_mapping
from auto_round.utils.model import cast_model_dtype, rename_weights_files


class DiffusionMixin:
    """Diffusion-specific functionality mixin.

    This mixin adds diffusion model-specific functionality to any compressor
    (Compressor, ImatrixCompressor, etc). It handles
    diffusion models (like Stable Diffusion, FLUX) that require special pipeline
    handling and data generation logic.

    Can be combined with:
    - Compressor (for AutoRound with calibration, or basic RTN)
    - ImatrixCompressor (for RTN with importance matrix)

    Diffusion-specific parameters:
        guidance_scale: Control how much image generation follows text prompt
        num_inference_steps: Number of denoising steps for diffusion generation or evaluation
        calib_num_inference_steps: Number of denoising steps used to collect calibration inputs
        generator_seed: Seed for initial noise generation
        diffusion_tuning_cache_size: Extra persistent GPU buffer budget in GiB for
            single-CUDA SignRound prefetch with low_gpu_mem_usage; 0 disables it.
            "auto" selects a conservative budget after the first tuning iteration.
            Training activations/workspace are not included in this budget.

    Design note:
        ``ModelContext._load_model()`` loads the diffusion pipeline and sets
        ``model_context.pipe`` and ``model_context.model`` (the unet/transformer).
        This mixin reads ``self.model_context.pipe`` directly during calibration and
        saving so that ``model_context`` remains the single source of truth.
    """

    def __init__(
        self,
        *args,
        guidance_scale: float = 7.5,
        num_inference_steps: int = 50,
        calib_num_inference_steps: int = 8,
        generator_seed: int | None = None,
        diffusion_tuning_cache_size: float | str = 0,
        **kwargs,
    ) -> None:
        if num_inference_steps < 1:
            raise ValueError("num_inference_steps must be a positive integer.")
        if calib_num_inference_steps < 1:
            raise ValueError("calib_num_inference_steps must be a positive integer.")
        if diffusion_tuning_cache_size != "auto":
            try:
                diffusion_tuning_cache_size = float(diffusion_tuning_cache_size)
                if not math.isfinite(diffusion_tuning_cache_size) or diffusion_tuning_cache_size < 0:
                    raise ValueError
            except (TypeError, ValueError):
                raise ValueError(
                    "diffusion_tuning_cache_size must be 'auto' or finite and non-negative (GiB)."
                ) from None

        self._diffusion_user_block_selector = copy.deepcopy(kwargs.get("to_quant_block_names"))
        self._diffusion_user_layer_config = copy.deepcopy(kwargs.get("layer_config"))

        # Store diffusion-specific attributes
        self.guidance_scale = guidance_scale
        self.num_inference_steps = num_inference_steps
        self.calib_num_inference_steps = calib_num_inference_steps
        self.generator_seed = generator_seed
        self.pipeline_call_kwargs = dict(kwargs.pop("pipeline_call_kwargs", {}) or {})

        # Use video-domain calibration prompts by default for diffusion models.
        if kwargs.get("dataset") in (None, "NeelNanda/pile-10k"):
            kwargs["dataset"] = "opens2v"

        iters = kwargs.get("iters", None)
        _alg_cfg = args[0] if args else None
        if iters is None and _alg_cfg is not None:
            cfgs = _alg_cfg if isinstance(_alg_cfg, list) else [_alg_cfg]
            for cfg in cfgs:
                if hasattr(cfg, "iters") and cfg.iters is not None:
                    iters = cfg.iters
                    break
        if iters is None:
            iters = 200

        if iters > 0:
            # ``batch_size`` is owned by the compressor / shared
            # CalibrationContext; it only comes from kwargs now (entry.py forwards
            # it explicitly).  AlgConfig no longer carries it.  Treat a missing
            # value as ``BaseCompressor``'s default (8) so the reset path always
            # triggers when the user didn't explicitly opt out.
            batch_size = kwargs.get("batch_size", 8)
            if batch_size != 1:
                grad_acc = kwargs.get("gradient_accumulate_steps", 1)
                if _alg_cfg is not None:
                    cfgs = _alg_cfg if isinstance(_alg_cfg, list) else [_alg_cfg]
                    for cfg in cfgs:
                        if hasattr(cfg, "gradient_accumulate_steps") and cfg.gradient_accumulate_steps is not None:
                            grad_acc = cfg.gradient_accumulate_steps
                            break
                new_grad_acc = batch_size * grad_acc
                kwargs["gradient_accumulate_steps"] = new_grad_acc
                kwargs["batch_size"] = 1
                if _alg_cfg is not None:
                    cfgs = _alg_cfg if isinstance(_alg_cfg, list) else [_alg_cfg]
                    for cfg in cfgs:
                        if hasattr(cfg, "gradient_accumulate_steps"):
                            cfg.gradient_accumulate_steps = new_grad_acc
                logger.warning(
                    f"reset batch_size({batch_size}) to 1 and "
                    f"gradient_accumulate_steps to {new_grad_acc} "
                    f"because batch_size={batch_size} cannot be used for calibrating non-text modules."
                )

        # Call parent class __init__ (will be Compressor, ImatrixCompressor, etc)
        super().__init__(*args, **kwargs)
        self.model_context.diffusion_tuning_cache_size = diffusion_tuning_cache_size

        pipe = getattr(self.model_context, "pipe", None)
        model = getattr(self.model_context, "model", None)
        if (
            getattr(self.model_context, "preloaded_diffusion_pipeline", False)
            and pipe is not None
            and model is not None
        ):
            self._align_pipeline_dtype(pipe, model.dtype)

    @staticmethod
    def _align_pipeline_dtype(pipe, target_dtype) -> None:
        """Align a preloaded pipeline without casting declared FP32 tensors."""

        for component_name in pipe.components:
            component = getattr(pipe, component_name, None)
            if not isinstance(component, torch.nn.Module):
                continue

            cast_model_dtype(component, target_dtype)

    def _get_calibrator_kind(self) -> str:
        """Select the diffusion calibration strategy.

        ``DiffusionCalibrator`` lives at
        :mod:`auto_round.calibration.diffusion` and owns diffusion-specific
        calibration input collection.
        """
        return "diffusion"

    def _find_additional_transformers(self):
        """Find transformer components beyond the primary one (e.g. transformer_2 in WAN)."""
        pipe = getattr(self.model_context, "pipe", None)
        if pipe is None:
            return []
        result = []
        for comp_name in pipe.components:
            comp = getattr(pipe, comp_name, None)
            if (
                comp_name.startswith("transformer")
                and comp_name != "transformer"
                and comp is not None
                and isinstance(comp, torch.nn.Module)
            ):
                result.append((comp_name, comp))
        return result

    def _defer_multi_transformer_serialization(self) -> None:
        """Keep every transformer executable until all calibration passes finish.

        Immediate packing replaces tuned ``nn.Linear`` modules with export-only
        ``QuantLinear`` modules. A multi-transformer pipeline such as WAN can
        still execute the primary transformer while collecting inputs for
        ``transformer_2``, so both packing and shard writing must be deferred.
        """
        self.compress_context.is_immediate_packing = False
        self.compress_context.is_immediate_saving = False

    def _align_device_and_dtype_for_secondary(self, transformer_name: str):
        """Dispatch a secondary transformer without changing component dtypes."""
        pipe = getattr(self.model_context, "pipe", None)
        model = getattr(self.model_context, "model", None)
        if pipe is None or model is None:
            return

        # Dispatch secondary transformer to GPU(s)
        device_map = getattr(self.compress_context, "device_map", None)
        device_list = getattr(self.compress_context, "device_list", [])
        multi_device = is_auto_device_mapping(device_map) and len(device_list) > 1

        if multi_device:
            comp_device = device_list[-1]
            for comp_name in pipe.components:
                comp = getattr(pipe, comp_name, None)
                if comp is None or comp is model or not hasattr(comp, "to"):
                    continue
                is_other_transformer = (
                    comp_name.startswith("transformer")
                    and isinstance(comp, torch.nn.Module)
                    and next(comp.parameters()).device.type == "cpu"
                )
                is_other_component = not comp_name.startswith("transformer")
                if is_other_transformer or is_other_component:
                    try:
                        comp.to(comp_device)
                    except (NotImplementedError, RuntimeError):
                        continue

            self.model_context.model = dispatch_model_block_wise(model, device_map)
            setattr(pipe, transformer_name, self.model_context.model)
        elif device_map is not None:
            target_device = get_major_device(device_map)
            pipe.to(target_device)

    def try_cache_inter_data_gpucpu(self, *args, **kwargs) -> Any:
        """Skip re-caching when DiffusionMixin.quantize has already populated self.inputs.

        CalibCompressor.quantize() always calls try_cache_inter_data_gpucpu, but for
        diffusion models the inputs were already collected by the diffusion pipeline
        in DiffusionMixin.quantize().  Return the cached data directly.
        """
        if getattr(self, "_inputs_cached", False):
            self._inputs_cached = False
            return self.inputs
        if hasattr(super(), "try_cache_inter_data_gpucpu"):
            return super().try_cache_inter_data_gpucpu(*args, **kwargs)

    def quantize(self) -> tuple[torch.nn.Module, dict]:
        """Quantize the diffusion model.

        Overrides the parent to use diffusion-specific cache_inter_data instead of
        the LLM-specific calib path.  The diffusion pipeline forward is used to collect
        block inputs (via _replace_forward hooks), then those inputs are passed to the
        standard CalibCompressor quantization loop.

        For dual-transformer pipelines (e.g. WAN with transformer + transformer_2),
        this method quantizes all transformers sequentially.
        """
        from auto_round.utils import find_matching_blocks, get_block_names
        from auto_round.utils.common import flatten_list

        self.post_init()

        # Zero-shot (RTN) path: no calibration data needed
        if not self.need_calib:
            return self._quantize_all_transformers_zero_shot()

        # Get block names and call cache_inter_data to populate self.inputs
        if bool(self.quant_block_list):
            all_blocks = self.quant_block_list
        else:
            all_blocks = get_block_names(self.model_context.model)
        if len(all_blocks) == 0:
            logger.warning("could not find blocks, exit with original model")
            return self.model, self.layer_config

        if not self.has_variable_block_shape:
            to_cache_block_names = [block[0] for block in all_blocks]
        else:
            to_cache_block_names = flatten_list(all_blocks)

        # Check for additional transformers (e.g. transformer_2 in WAN)
        additional = self._find_additional_transformers()
        if not additional:
            # Single-transformer path: let calib() own pipeline dispatch.
            pipe = self.model_context.pipe
            device_map = getattr(self.compress_context, "device_map", None)
            if device_map is not None and not is_auto_device_mapping(device_map) and not isinstance(device_map, int):
                target_device = get_major_device(device_map)
                # Skip if the transformer is already on the target device to avoid
                # redundant full-model transfer that exhausts GPU memory.
                transformer = getattr(pipe, "transformer", None) or getattr(pipe, "unet", None)
                param = next(transformer.parameters(), None) if transformer else None
                skip_move = param is not None and get_major_device(str(param.device)) == target_device
                if not skip_move:
                    pipe.to(target_device)

            logger.info("start to cache block inputs")
            all_inputs = self.try_cache_inter_data_gpucpu(
                to_cache_block_names,
                self.calibration_context.nsamples,
                layer_names=[],
            )
            self.inputs = all_inputs
            if getattr(self.calibration, "_cpu_offload_mode", None) == "model":
                from accelerate.hooks import remove_hook_from_submodules

                remove_hook_from_submodules(self.model_context.model)
            clear_memory()
            self._inputs_cached = True
            return super().quantize()

        # Dual-transformer path: quantize all transformers sequentially
        logger.info("Detected multi-transformer diffusion pipeline, quantizing all transformers")

        # Ensure at least 2 calibration inference steps so both transformers are exercised.
        orig_steps = getattr(self, "calib_num_inference_steps", None) or 8
        if orig_steps < 2:
            logger.warning(
                f"calib_num_inference_steps={orig_steps} is too low for dual-transformer "
                f"quantization — increasing to 2 so all transformers receive calibration data."
            )
            self.calib_num_inference_steps = 2

        # Disable low_cpu_mem_usage so quantized models stay in memory during multi-transformer
        # quantization.
        orig_low_cpu = self.compress_context.low_cpu_mem_usage
        orig_immediate_packing = self.compress_context.is_immediate_packing
        orig_immediate_saving = self.compress_context.is_immediate_saving
        self.compress_context.low_cpu_mem_usage = False
        # Keep the primary transformer executable for the later transformer_2
        # calibration pass. The llm_compressor QuantLinear produced by immediate
        # packing is an export container and intentionally has no forward method.
        self._defer_multi_transformer_serialization()

        # Store primary transformer state
        primary_model = self.model
        primary_layer_config = dict(self.layer_config) if self.layer_config else {}
        primary_quant_block_list = list(self.quant_block_list) if self.quant_block_list else []
        quantized_extras = {}

        # Quantize primary transformer
        logger.info("start to cache block inputs for primary transformer")
        all_inputs = self.try_cache_inter_data_gpucpu(
            to_cache_block_names,
            self.calibration_context.nsamples,
            layer_names=[],
        )
        self.inputs = all_inputs
        clear_memory(device_list=device_manager.device_list)
        self._inputs_cached = True
        super().quantize()

        # Clear stale hf_device_map so cache_inter_data can re-dispatch for next transformer
        if hasattr(primary_model, "hf_device_map"):
            delattr(primary_model, "hf_device_map")

        # Quantize additional transformers
        for comp_name, transformer in additional:
            logger.info(f"Quantizing {comp_name}")

            # Reset quantization state for new transformer
            self.model_context.model = transformer
            self.model_context.quantized = False
            self._post_init_done = False
            # Calibrators snapshot the active model at construction time. Recreate
            # it so transformer_2 block hooks are not installed on the primary model.
            self.calibration = None

            # Dispatch the pipeline for the secondary transformer without recasting it.
            self._align_device_and_dtype_for_secondary(comp_name)

            # Re-run post_init to set up quantizer for new model
            self.post_init()
            # post_init recomputes immediate packing/saving from the output format.
            # Reassert the multi-transformer deferral before pipeline calibration.
            self._defer_multi_transformer_serialization()

            # Get block names for new transformer
            all_blocks = get_block_names(self.model_context.model)
            self.quant_block_list = find_matching_blocks(self.model_context.model, all_blocks, None)

            # Get new block names for caching
            if bool(self.quant_block_list):
                all_blocks = self.quant_block_list
            else:
                all_blocks = get_block_names(self.model_context.model)
            if len(all_blocks) == 0:
                logger.warning(f"could not find blocks in {comp_name}, skipping")
                continue

            if not self.has_variable_block_shape:
                to_cache_block_names = [block[0] for block in all_blocks]
            else:
                to_cache_block_names = flatten_list(all_blocks)

            logger.info(f"start to cache block inputs for {comp_name}")
            all_inputs = self.try_cache_inter_data_gpucpu(
                to_cache_block_names,
                self.calibration_context.nsamples,
                layer_names=[],
            )
            self.inputs = all_inputs
            clear_memory(device_list=device_manager.device_list)
            self._inputs_cached = True
            super().quantize()

            # Store quantized transformer
            quantized_extras[comp_name] = (self.model_context.model, dict(self.layer_config))
            # Also update the pipeline to reference the quantized transformer
            setattr(self.model_context.pipe, comp_name, self.model_context.model)

        # Restore primary transformer state
        self.model_context.model = primary_model
        self.model_context.quantized = True
        self.layer_config = primary_layer_config
        self.quant_block_list = primary_quant_block_list
        self._quantized_transformers = quantized_extras
        self.compress_context.low_cpu_mem_usage = orig_low_cpu
        self.compress_context.is_immediate_packing = orig_immediate_packing
        self.compress_context.is_immediate_saving = orig_immediate_saving
        self.calib_num_inference_steps = orig_steps

        return self.model_context.model, self.layer_config

    def _quantize_all_transformers_zero_shot(self):
        """Quantize every transformer without running pipeline calibration."""
        from auto_round.utils import find_matching_blocks, get_block_names

        additional = self._find_additional_transformers()
        if not additional:
            return self._quantize_zero_shot()

        primary_model = self.model_context.model
        primary_blocks = self.quant_block_list
        primary_config = self.layer_config
        primary_quantized = self.model_context.quantized
        state_keys = (
            "compression_plan",
            "_format_resolution",
            "_alg_composer",
            "to_quant_block_names",
            "ignore_layers",
            "calibration",
            "_post_init_done",
        )
        primary_state = {key: getattr(self, key, None) for key in state_keys}
        context = self.compress_context
        original_flags = (context.low_cpu_mem_usage, context.is_immediate_packing, context.is_immediate_saving)
        context.low_cpu_mem_usage = False
        self._defer_multi_transformer_serialization()
        extras = {}
        plans = {}
        try:
            self._quantize_zero_shot()
            primary_quantized = True
            primary_config = dict(self.layer_config)
            primary_state["compression_plan"] = getattr(self, "compression_plan", None)
            for name, transformer in additional:
                logger.info(f"Quantizing {name} without calibration")
                self.model_context.model = transformer
                self.model_context.quantized = False
                self._post_init_done = False
                self.calibration = None
                self.to_quant_block_names = copy.deepcopy(self._diffusion_user_block_selector)
                self.quant_block_list = find_matching_blocks(
                    transformer, get_block_names(transformer), self.to_quant_block_names
                )
                # Build the component's plan from user overrides, not the primary's expanded plan.
                self.layer_config = copy.deepcopy(self._diffusion_user_layer_config) or {}
                self.ignore_layers = primary_state["ignore_layers"]
                resolution = getattr(self, "_format_resolution", None)
                if resolution is not None:
                    self._format_resolution = replace(resolution, quant_block_list=self.quant_block_list)
                self.post_init()
                self._defer_multi_transformer_serialization()
                self._quantize_zero_shot()
                extras[name] = (self.model_context.model, dict(self.layer_config))
                plans[name] = getattr(self, "compression_plan", None)
                setattr(self.model_context.pipe, name, self.model_context.model)
        finally:
            self.model_context.model = primary_model
            for key, value in primary_state.items():
                setattr(self, key, value)
            self.layer_config = primary_config
            self.quant_block_list = primary_blocks
            self.model_context.quantized = primary_quantized
            self._quantized_transformers = extras
            self._quantized_transformer_plans = plans
            context.low_cpu_mem_usage, context.is_immediate_packing, context.is_immediate_saving = original_flags
        return primary_model, self.layer_config

    def save_quantized(
        self,
        output_dir: str | None = None,
        format: str | list | None = None,
        inplace: bool = True,
        return_folders: bool = False,
        **kwargs,
    ) -> Any:
        """Save the quantized model to the specified output directory in the specified format.

        For multi-transformer pipelines, all quantized transformers are saved.

        Args:
            output_dir (str, optional): The directory to save the quantized model. Defaults to None.
            format (str, optional): The format in which to save the model. Defaults to "auto_round".
            inplace (bool, optional): Whether to modify the model in place. Defaults to True.
            return_folders (bool, optional): Whether to return the save folder paths. Defaults to False.
            **kwargs: Additional keyword arguments specific to the export format.

        Returns:
            object: The compressed model object, or (compressed_model, folders) if return_folders is True.
        """
        if output_dir is None:
            return super().save_quantized(
                output_dir, format=format, inplace=inplace, return_folders=return_folders, **kwargs
            )

        pipe = self.model_context.pipe
        quantized_transformers = getattr(self, "_quantized_transformers", {})
        compressed_model = None
        folders = []
        has_multiple_quantized_transformers = bool(quantized_transformers)

        # Handle multi-format (convert string to list if needed)
        _format = format if format is not None else getattr(self, "formats", None) or "auto_round"
        if isinstance(_format, str):
            _format = self._resolve_format_string(_format)

        for name in pipe.components.keys():
            val = getattr(pipe, name)
            sub_module_path = (
                os.path.join(output_dir, name) if os.path.basename(os.path.normpath(output_dir)) != name else output_dir
            )
            target_output_dir = (
                sub_module_path
                if has_multiple_quantized_transformers or not self.compress_context.is_immediate_saving
                else output_dir
            )
            if name in quantized_transformers:
                # Save secondary quantized transformer
                saved_model = self.model_context.model
                saved_lc = self.layer_config
                saved_plan = getattr(self, "compression_plan", None)
                saved_selector = self.to_quant_block_names
                component_plan = getattr(self, "_quantized_transformer_plans", {}).get(name)
                if component_plan is not None:
                    self.compression_plan = component_plan
                    # None lets serialization derive the selector from this component's plan.
                    self.to_quant_block_names = copy.deepcopy(self._diffusion_user_block_selector)
                saved_immediate_saving = self.compress_context.is_immediate_saving
                self.model_context.model, self.layer_config = quantized_transformers[name]
                saved_subfolder = getattr(self.model_context.model, "_autoround_pipeline_subfolder", None)
                if has_multiple_quantized_transformers:
                    self.compress_context.is_immediate_saving = False
                    if saved_subfolder is not None:
                        delattr(self.model_context.model, "_autoround_pipeline_subfolder")
                try:
                    compressed_model = super().save_quantized(
                        output_dir=target_output_dir,
                        format=_format,
                        inplace=inplace,
                        return_folders=False,
                        **kwargs,
                    )
                finally:
                    self.compress_context.is_immediate_saving = saved_immediate_saving
                    if saved_subfolder is not None:
                        self.model_context.model._autoround_pipeline_subfolder = saved_subfolder
                    self.model_context.model = saved_model
                    self.compression_plan = saved_plan
                    self.to_quant_block_names = saved_selector
                    self.layer_config = saved_lc
            elif val is self.model_context.model:
                # Save primary quantized transformer
                saved_immediate_saving = self.compress_context.is_immediate_saving
                saved_subfolder = getattr(self.model_context.model, "_autoround_pipeline_subfolder", None)
                if has_multiple_quantized_transformers:
                    self.compress_context.is_immediate_saving = False
                    if saved_subfolder is not None:
                        delattr(self.model_context.model, "_autoround_pipeline_subfolder")
                compressed_model = super().save_quantized(
                    output_dir=target_output_dir,
                    format=_format,
                    inplace=inplace,
                    return_folders=False,
                    **kwargs,
                )
                self.compress_context.is_immediate_saving = saved_immediate_saving
                if saved_subfolder is not None:
                    self.model_context.model._autoround_pipeline_subfolder = saved_subfolder
            elif val is not None and hasattr(val, "save_pretrained"):
                if (
                    pipe.__class__.__name__ == "ZImagePipeline"
                    and name == "text_encoder"
                    and val.__class__.__name__ == "Qwen3Model"
                ):
                    # Preserve the original causal-LM layout expected by vLLM-Omni.
                    state_dict = {f"model.{key}": value for key, value in val.state_dict().items()}
                    val.save_pretrained(sub_module_path, state_dict=state_dict)
                    config = copy.deepcopy(val.config)
                    config.architectures = ["Qwen3ForCausalLM"]
                    config.save_pretrained(sub_module_path)
                else:
                    val.save_pretrained(sub_module_path)
                continue

            if name.startswith("transformer"):
                rename_weights_files(target_output_dir)

            folders.append(target_output_dir)

        if hasattr(pipe, "save_config"):
            pipe.save_config(output_dir)
        elif hasattr(pipe.config, "save_pretrained"):
            pipe.config.save_pretrained(output_dir)
        else:
            # FrozenDict / plain dict — write model_index.json manually
            model_index_path = os.path.join(output_dir, "model_index.json")
            with open(model_index_path, "w", encoding="utf-8") as f:
                f.write(json.dumps(dict(pipe.config), indent=2, sort_keys=True) + "\n")

        if return_folders:
            return compressed_model, folders
        return compressed_model