PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
# Copyright (c) 2026 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import copy
import json
import math
import os
from dataclasses import replace
from typing import Any
import torch
from auto_round.logger import logger
from auto_round.utils import clear_memory
from auto_round.utils.device import (
dispatch_model_block_wise,
get_major_device,
)
from auto_round.utils.device_manager import device_manager, is_auto_device_mapping
from auto_round.utils.model import cast_model_dtype, rename_weights_files
class DiffusionMixin:
"""Diffusion-specific functionality mixin.
This mixin adds diffusion model-specific functionality to any compressor
(Compressor, ImatrixCompressor, etc). It handles
diffusion models (like Stable Diffusion, FLUX) that require special pipeline
handling and data generation logic.
Can be combined with:
- Compressor (for AutoRound with calibration, or basic RTN)
- ImatrixCompressor (for RTN with importance matrix)
Diffusion-specific parameters:
guidance_scale: Control how much image generation follows text prompt
num_inference_steps: Number of denoising steps for diffusion generation or evaluation
calib_num_inference_steps: Number of denoising steps used to collect calibration inputs
generator_seed: Seed for initial noise generation
diffusion_tuning_cache_size: Extra persistent GPU buffer budget in GiB for
single-CUDA SignRound prefetch with low_gpu_mem_usage; 0 disables it.
"auto" selects a conservative budget after the first tuning iteration.
Training activations/workspace are not included in this budget.
Design note:
``ModelContext._load_model()`` loads the diffusion pipeline and sets
``model_context.pipe`` and ``model_context.model`` (the unet/transformer).
This mixin reads ``self.model_context.pipe`` directly during calibration and
saving so that ``model_context`` remains the single source of truth.
"""
def __init__(
self,
*args,
guidance_scale: float = 7.5,
num_inference_steps: int = 50,
calib_num_inference_steps: int = 8,
generator_seed: int | None = None,
diffusion_tuning_cache_size: float | str = 0,
**kwargs,
) -> None:
if num_inference_steps < 1:
raise ValueError("num_inference_steps must be a positive integer.")
if calib_num_inference_steps < 1:
raise ValueError("calib_num_inference_steps must be a positive integer.")
if diffusion_tuning_cache_size != "auto":
try:
diffusion_tuning_cache_size = float(diffusion_tuning_cache_size)
if not math.isfinite(diffusion_tuning_cache_size) or diffusion_tuning_cache_size < 0:
raise ValueError
except (TypeError, ValueError):
raise ValueError(
"diffusion_tuning_cache_size must be 'auto' or finite and non-negative (GiB)."
) from None
self._diffusion_user_block_selector = copy.deepcopy(kwargs.get("to_quant_block_names"))
self._diffusion_user_layer_config = copy.deepcopy(kwargs.get("layer_config"))
# Store diffusion-specific attributes
self.guidance_scale = guidance_scale
self.num_inference_steps = num_inference_steps
self.calib_num_inference_steps = calib_num_inference_steps
self.generator_seed = generator_seed
self.pipeline_call_kwargs = dict(kwargs.pop("pipeline_call_kwargs", {}) or {})
# Use video-domain calibration prompts by default for diffusion models.
if kwargs.get("dataset") in (None, "NeelNanda/pile-10k"):
kwargs["dataset"] = "opens2v"
iters = kwargs.get("iters", None)
_alg_cfg = args[0] if args else None
if iters is None and _alg_cfg is not None:
cfgs = _alg_cfg if isinstance(_alg_cfg, list) else [_alg_cfg]
for cfg in cfgs:
if hasattr(cfg, "iters") and cfg.iters is not None:
iters = cfg.iters
break
if iters is None:
iters = 200
if iters > 0:
# ``batch_size`` is owned by the compressor / shared
# CalibrationContext; it only comes from kwargs now (entry.py forwards
# it explicitly). AlgConfig no longer carries it. Treat a missing
# value as ``BaseCompressor``'s default (8) so the reset path always
# triggers when the user didn't explicitly opt out.
batch_size = kwargs.get("batch_size", 8)
if batch_size != 1:
grad_acc = kwargs.get("gradient_accumulate_steps", 1)
if _alg_cfg is not None:
cfgs = _alg_cfg if isinstance(_alg_cfg, list) else [_alg_cfg]
for cfg in cfgs:
if hasattr(cfg, "gradient_accumulate_steps") and cfg.gradient_accumulate_steps is not None:
grad_acc = cfg.gradient_accumulate_steps
break
new_grad_acc = batch_size * grad_acc
kwargs["gradient_accumulate_steps"] = new_grad_acc
kwargs["batch_size"] = 1
if _alg_cfg is not None:
cfgs = _alg_cfg if isinstance(_alg_cfg, list) else [_alg_cfg]
for cfg in cfgs:
if hasattr(cfg, "gradient_accumulate_steps"):
cfg.gradient_accumulate_steps = new_grad_acc
logger.warning(
f"reset batch_size({batch_size}) to 1 and "
f"gradient_accumulate_steps to {new_grad_acc} "
f"because batch_size={batch_size} cannot be used for calibrating non-text modules."
)
# Call parent class __init__ (will be Compressor, ImatrixCompressor, etc)
super().__init__(*args, **kwargs)
self.model_context.diffusion_tuning_cache_size = diffusion_tuning_cache_size
pipe = getattr(self.model_context, "pipe", None)
model = getattr(self.model_context, "model", None)
if (
getattr(self.model_context, "preloaded_diffusion_pipeline", False)
and pipe is not None
and model is not None
):
self._align_pipeline_dtype(pipe, model.dtype)
@staticmethod
def _align_pipeline_dtype(pipe, target_dtype) -> None:
"""Align a preloaded pipeline without casting declared FP32 tensors."""
for component_name in pipe.components:
component = getattr(pipe, component_name, None)
if not isinstance(component, torch.nn.Module):
continue
cast_model_dtype(component, target_dtype)
def _get_calibrator_kind(self) -> str:
"""Select the diffusion calibration strategy.
``DiffusionCalibrator`` lives at
:mod:`auto_round.calibration.diffusion` and owns diffusion-specific
calibration input collection.
"""
return "diffusion"
def _find_additional_transformers(self):
"""Find transformer components beyond the primary one (e.g. transformer_2 in WAN)."""
pipe = getattr(self.model_context, "pipe", None)
if pipe is None:
return []
result = []
for comp_name in pipe.components:
comp = getattr(pipe, comp_name, None)
if (
comp_name.startswith("transformer")
and comp_name != "transformer"
and comp is not None
and isinstance(comp, torch.nn.Module)
):
result.append((comp_name, comp))
return result
def _defer_multi_transformer_serialization(self) -> None:
"""Keep every transformer executable until all calibration passes finish.
Immediate packing replaces tuned ``nn.Linear`` modules with export-only
``QuantLinear`` modules. A multi-transformer pipeline such as WAN can
still execute the primary transformer while collecting inputs for
``transformer_2``, so both packing and shard writing must be deferred.
"""
self.compress_context.is_immediate_packing = False
self.compress_context.is_immediate_saving = False
def _align_device_and_dtype_for_secondary(self, transformer_name: str):
"""Dispatch a secondary transformer without changing component dtypes."""
pipe = getattr(self.model_context, "pipe", None)
model = getattr(self.model_context, "model", None)
if pipe is None or model is None:
return
# Dispatch secondary transformer to GPU(s)
device_map = getattr(self.compress_context, "device_map", None)
device_list = getattr(self.compress_context, "device_list", [])
multi_device = is_auto_device_mapping(device_map) and len(device_list) > 1
if multi_device:
comp_device = device_list[-1]
for comp_name in pipe.components:
comp = getattr(pipe, comp_name, None)
if comp is None or comp is model or not hasattr(comp, "to"):
continue
is_other_transformer = (
comp_name.startswith("transformer")
and isinstance(comp, torch.nn.Module)
and next(comp.parameters()).device.type == "cpu"
)
is_other_component = not comp_name.startswith("transformer")
if is_other_transformer or is_other_component:
try:
comp.to(comp_device)
except (NotImplementedError, RuntimeError):
continue
self.model_context.model = dispatch_model_block_wise(model, device_map)
setattr(pipe, transformer_name, self.model_context.model)
elif device_map is not None:
target_device = get_major_device(device_map)
pipe.to(target_device)
def try_cache_inter_data_gpucpu(self, *args, **kwargs) -> Any:
"""Skip re-caching when DiffusionMixin.quantize has already populated self.inputs.
CalibCompressor.quantize() always calls try_cache_inter_data_gpucpu, but for
diffusion models the inputs were already collected by the diffusion pipeline
in DiffusionMixin.quantize(). Return the cached data directly.
"""
if getattr(self, "_inputs_cached", False):
self._inputs_cached = False
return self.inputs
if hasattr(super(), "try_cache_inter_data_gpucpu"):
return super().try_cache_inter_data_gpucpu(*args, **kwargs)
def quantize(self) -> tuple[torch.nn.Module, dict]:
"""Quantize the diffusion model.
Overrides the parent to use diffusion-specific cache_inter_data instead of
the LLM-specific calib path. The diffusion pipeline forward is used to collect
block inputs (via _replace_forward hooks), then those inputs are passed to the
standard CalibCompressor quantization loop.
For dual-transformer pipelines (e.g. WAN with transformer + transformer_2),
this method quantizes all transformers sequentially.
"""
from auto_round.utils import find_matching_blocks, get_block_names
from auto_round.utils.common import flatten_list
self.post_init()
# Zero-shot (RTN) path: no calibration data needed
if not self.need_calib:
return self._quantize_all_transformers_zero_shot()
# Get block names and call cache_inter_data to populate self.inputs
if bool(self.quant_block_list):
all_blocks = self.quant_block_list
else:
all_blocks = get_block_names(self.model_context.model)
if len(all_blocks) == 0:
logger.warning("could not find blocks, exit with original model")
return self.model, self.layer_config
if not self.has_variable_block_shape:
to_cache_block_names = [block[0] for block in all_blocks]
else:
to_cache_block_names = flatten_list(all_blocks)
# Check for additional transformers (e.g. transformer_2 in WAN)
additional = self._find_additional_transformers()
if not additional:
# Single-transformer path: let calib() own pipeline dispatch.
pipe = self.model_context.pipe
device_map = getattr(self.compress_context, "device_map", None)
if device_map is not None and not is_auto_device_mapping(device_map) and not isinstance(device_map, int):
target_device = get_major_device(device_map)
# Skip if the transformer is already on the target device to avoid
# redundant full-model transfer that exhausts GPU memory.
transformer = getattr(pipe, "transformer", None) or getattr(pipe, "unet", None)
param = next(transformer.parameters(), None) if transformer else None
skip_move = param is not None and get_major_device(str(param.device)) == target_device
if not skip_move:
pipe.to(target_device)
logger.info("start to cache block inputs")
all_inputs = self.try_cache_inter_data_gpucpu(
to_cache_block_names,
self.calibration_context.nsamples,
layer_names=[],
)
self.inputs = all_inputs
if getattr(self.calibration, "_cpu_offload_mode", None) == "model":
from accelerate.hooks import remove_hook_from_submodules
remove_hook_from_submodules(self.model_context.model)
clear_memory()
self._inputs_cached = True
return super().quantize()
# Dual-transformer path: quantize all transformers sequentially
logger.info("Detected multi-transformer diffusion pipeline, quantizing all transformers")
# Ensure at least 2 calibration inference steps so both transformers are exercised.
orig_steps = getattr(self, "calib_num_inference_steps", None) or 8
if orig_steps < 2:
logger.warning(
f"calib_num_inference_steps={orig_steps} is too low for dual-transformer "
f"quantization — increasing to 2 so all transformers receive calibration data."
)
self.calib_num_inference_steps = 2
# Disable low_cpu_mem_usage so quantized models stay in memory during multi-transformer
# quantization.
orig_low_cpu = self.compress_context.low_cpu_mem_usage
orig_immediate_packing = self.compress_context.is_immediate_packing
orig_immediate_saving = self.compress_context.is_immediate_saving
self.compress_context.low_cpu_mem_usage = False
# Keep the primary transformer executable for the later transformer_2
# calibration pass. The llm_compressor QuantLinear produced by immediate
# packing is an export container and intentionally has no forward method.
self._defer_multi_transformer_serialization()
# Store primary transformer state
primary_model = self.model
primary_layer_config = dict(self.layer_config) if self.layer_config else {}
primary_quant_block_list = list(self.quant_block_list) if self.quant_block_list else []
quantized_extras = {}
# Quantize primary transformer
logger.info("start to cache block inputs for primary transformer")
all_inputs = self.try_cache_inter_data_gpucpu(
to_cache_block_names,
self.calibration_context.nsamples,
layer_names=[],
)
self.inputs = all_inputs
clear_memory(device_list=device_manager.device_list)
self._inputs_cached = True
super().quantize()
# Clear stale hf_device_map so cache_inter_data can re-dispatch for next transformer
if hasattr(primary_model, "hf_device_map"):
delattr(primary_model, "hf_device_map")
# Quantize additional transformers
for comp_name, transformer in additional:
logger.info(f"Quantizing {comp_name}")
# Reset quantization state for new transformer
self.model_context.model = transformer
self.model_context.quantized = False
self._post_init_done = False
# Calibrators snapshot the active model at construction time. Recreate
# it so transformer_2 block hooks are not installed on the primary model.
self.calibration = None
# Dispatch the pipeline for the secondary transformer without recasting it.
self._align_device_and_dtype_for_secondary(comp_name)
# Re-run post_init to set up quantizer for new model
self.post_init()
# post_init recomputes immediate packing/saving from the output format.
# Reassert the multi-transformer deferral before pipeline calibration.
self._defer_multi_transformer_serialization()
# Get block names for new transformer
all_blocks = get_block_names(self.model_context.model)
self.quant_block_list = find_matching_blocks(self.model_context.model, all_blocks, None)
# Get new block names for caching
if bool(self.quant_block_list):
all_blocks = self.quant_block_list
else:
all_blocks = get_block_names(self.model_context.model)
if len(all_blocks) == 0:
logger.warning(f"could not find blocks in {comp_name}, skipping")
continue
if not self.has_variable_block_shape:
to_cache_block_names = [block[0] for block in all_blocks]
else:
to_cache_block_names = flatten_list(all_blocks)
logger.info(f"start to cache block inputs for {comp_name}")
all_inputs = self.try_cache_inter_data_gpucpu(
to_cache_block_names,
self.calibration_context.nsamples,
layer_names=[],
)
self.inputs = all_inputs
clear_memory(device_list=device_manager.device_list)
self._inputs_cached = True
super().quantize()
# Store quantized transformer
quantized_extras[comp_name] = (self.model_context.model, dict(self.layer_config))
# Also update the pipeline to reference the quantized transformer
setattr(self.model_context.pipe, comp_name, self.model_context.model)
# Restore primary transformer state
self.model_context.model = primary_model
self.model_context.quantized = True
self.layer_config = primary_layer_config
self.quant_block_list = primary_quant_block_list
self._quantized_transformers = quantized_extras
self.compress_context.low_cpu_mem_usage = orig_low_cpu
self.compress_context.is_immediate_packing = orig_immediate_packing
self.compress_context.is_immediate_saving = orig_immediate_saving
self.calib_num_inference_steps = orig_steps
return self.model_context.model, self.layer_config
def _quantize_all_transformers_zero_shot(self):
"""Quantize every transformer without running pipeline calibration."""
from auto_round.utils import find_matching_blocks, get_block_names
additional = self._find_additional_transformers()
if not additional:
return self._quantize_zero_shot()
primary_model = self.model_context.model
primary_blocks = self.quant_block_list
primary_config = self.layer_config
primary_quantized = self.model_context.quantized
state_keys = (
"compression_plan",
"_format_resolution",
"_alg_composer",
"to_quant_block_names",
"ignore_layers",
"calibration",
"_post_init_done",
)
primary_state = {key: getattr(self, key, None) for key in state_keys}
context = self.compress_context
original_flags = (context.low_cpu_mem_usage, context.is_immediate_packing, context.is_immediate_saving)
context.low_cpu_mem_usage = False
self._defer_multi_transformer_serialization()
extras = {}
plans = {}
try:
self._quantize_zero_shot()
primary_quantized = True
primary_config = dict(self.layer_config)
primary_state["compression_plan"] = getattr(self, "compression_plan", None)
for name, transformer in additional:
logger.info(f"Quantizing {name} without calibration")
self.model_context.model = transformer
self.model_context.quantized = False
self._post_init_done = False
self.calibration = None
self.to_quant_block_names = copy.deepcopy(self._diffusion_user_block_selector)
self.quant_block_list = find_matching_blocks(
transformer, get_block_names(transformer), self.to_quant_block_names
)
# Build the component's plan from user overrides, not the primary's expanded plan.
self.layer_config = copy.deepcopy(self._diffusion_user_layer_config) or {}
self.ignore_layers = primary_state["ignore_layers"]
resolution = getattr(self, "_format_resolution", None)
if resolution is not None:
self._format_resolution = replace(resolution, quant_block_list=self.quant_block_list)
self.post_init()
self._defer_multi_transformer_serialization()
self._quantize_zero_shot()
extras[name] = (self.model_context.model, dict(self.layer_config))
plans[name] = getattr(self, "compression_plan", None)
setattr(self.model_context.pipe, name, self.model_context.model)
finally:
self.model_context.model = primary_model
for key, value in primary_state.items():
setattr(self, key, value)
self.layer_config = primary_config
self.quant_block_list = primary_blocks
self.model_context.quantized = primary_quantized
self._quantized_transformers = extras
self._quantized_transformer_plans = plans
context.low_cpu_mem_usage, context.is_immediate_packing, context.is_immediate_saving = original_flags
return primary_model, self.layer_config
def save_quantized(
self,
output_dir: str | None = None,
format: str | list | None = None,
inplace: bool = True,
return_folders: bool = False,
**kwargs,
) -> Any:
"""Save the quantized model to the specified output directory in the specified format.
For multi-transformer pipelines, all quantized transformers are saved.
Args:
output_dir (str, optional): The directory to save the quantized model. Defaults to None.
format (str, optional): The format in which to save the model. Defaults to "auto_round".
inplace (bool, optional): Whether to modify the model in place. Defaults to True.
return_folders (bool, optional): Whether to return the save folder paths. Defaults to False.
**kwargs: Additional keyword arguments specific to the export format.
Returns:
object: The compressed model object, or (compressed_model, folders) if return_folders is True.
"""
if output_dir is None:
return super().save_quantized(
output_dir, format=format, inplace=inplace, return_folders=return_folders, **kwargs
)
pipe = self.model_context.pipe
quantized_transformers = getattr(self, "_quantized_transformers", {})
compressed_model = None
folders = []
has_multiple_quantized_transformers = bool(quantized_transformers)
# Handle multi-format (convert string to list if needed)
_format = format if format is not None else getattr(self, "formats", None) or "auto_round"
if isinstance(_format, str):
_format = self._resolve_format_string(_format)
for name in pipe.components.keys():
val = getattr(pipe, name)
sub_module_path = (
os.path.join(output_dir, name) if os.path.basename(os.path.normpath(output_dir)) != name else output_dir
)
target_output_dir = (
sub_module_path
if has_multiple_quantized_transformers or not self.compress_context.is_immediate_saving
else output_dir
)
if name in quantized_transformers:
# Save secondary quantized transformer
saved_model = self.model_context.model
saved_lc = self.layer_config
saved_plan = getattr(self, "compression_plan", None)
saved_selector = self.to_quant_block_names
component_plan = getattr(self, "_quantized_transformer_plans", {}).get(name)
if component_plan is not None:
self.compression_plan = component_plan
# None lets serialization derive the selector from this component's plan.
self.to_quant_block_names = copy.deepcopy(self._diffusion_user_block_selector)
saved_immediate_saving = self.compress_context.is_immediate_saving
self.model_context.model, self.layer_config = quantized_transformers[name]
saved_subfolder = getattr(self.model_context.model, "_autoround_pipeline_subfolder", None)
if has_multiple_quantized_transformers:
self.compress_context.is_immediate_saving = False
if saved_subfolder is not None:
delattr(self.model_context.model, "_autoround_pipeline_subfolder")
try:
compressed_model = super().save_quantized(
output_dir=target_output_dir,
format=_format,
inplace=inplace,
return_folders=False,
**kwargs,
)
finally:
self.compress_context.is_immediate_saving = saved_immediate_saving
if saved_subfolder is not None:
self.model_context.model._autoround_pipeline_subfolder = saved_subfolder
self.model_context.model = saved_model
self.compression_plan = saved_plan
self.to_quant_block_names = saved_selector
self.layer_config = saved_lc
elif val is self.model_context.model:
# Save primary quantized transformer
saved_immediate_saving = self.compress_context.is_immediate_saving
saved_subfolder = getattr(self.model_context.model, "_autoround_pipeline_subfolder", None)
if has_multiple_quantized_transformers:
self.compress_context.is_immediate_saving = False
if saved_subfolder is not None:
delattr(self.model_context.model, "_autoround_pipeline_subfolder")
compressed_model = super().save_quantized(
output_dir=target_output_dir,
format=_format,
inplace=inplace,
return_folders=False,
**kwargs,
)
self.compress_context.is_immediate_saving = saved_immediate_saving
if saved_subfolder is not None:
self.model_context.model._autoround_pipeline_subfolder = saved_subfolder
elif val is not None and hasattr(val, "save_pretrained"):
if (
pipe.__class__.__name__ == "ZImagePipeline"
and name == "text_encoder"
and val.__class__.__name__ == "Qwen3Model"
):
# Preserve the original causal-LM layout expected by vLLM-Omni.
state_dict = {f"model.{key}": value for key, value in val.state_dict().items()}
val.save_pretrained(sub_module_path, state_dict=state_dict)
config = copy.deepcopy(val.config)
config.architectures = ["Qwen3ForCausalLM"]
config.save_pretrained(sub_module_path)
else:
val.save_pretrained(sub_module_path)
continue
if name.startswith("transformer"):
rename_weights_files(target_output_dir)
folders.append(target_output_dir)
if hasattr(pipe, "save_config"):
pipe.save_config(output_dir)
elif hasattr(pipe.config, "save_pretrained"):
pipe.config.save_pretrained(output_dir)
else:
# FrozenDict / plain dict — write model_index.json manually
model_index_path = os.path.join(output_dir, "model_index.json")
with open(model_index_path, "w", encoding="utf-8") as f:
f.write(json.dumps(dict(pipe.config), indent=2, sort_keys=True) + "\n")
if return_folders:
return compressed_model, folders
return compressed_model