Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

Feats: Quantize/save/evaluate the Wan-AI/WAN2.2 models in w4a16 format (#1678)

Liang Lvf1db02a3f50d
diff --git a/auto_round/algorithms/quantization/base.py b/auto_round/algorithms/quantization/base.py
index fd39f547e..b0e0cbf6b 100644
--- a/auto_round/algorithms/quantization/base.py
+++ b/auto_round/algorithms/quantization/base.py
@@ -53,6 +53,7 @@ class BaseQuantizers:
"FluxSingleTransformerBlock": ["encoder_hidden_states", "hidden_states"],
"OvisImageTransformerBlock": ["encoder_hidden_states", "hidden_states"],
"OvisImageSingleTransformerBlock": ["encoder_hidden_states", "hidden_states"],
+ "WanTransformerBlock": ["hidden_states"],
}
def __init__(self, config: QuantizationConfig):
@@ -368,21 +369,28 @@ class BaseQuantizers:
input_others,
bs: int,
save_output: bool = True,
+ device_override: Union[torch.device, str, None] = None,
):
"""Compute the output of a block for calibration inputs.
Shared by SignRoundQuantizer and OptimizedRTNQuantizer. Algorithm-specific
block-forward selection (compile vs. plain) is handled here based on
``enable_alg_ext`` and act-quantization flags.
+
+ Args:
+ device_override: Override the target device. Used by diffusion with
+ multi-device dispatch to pass None so block_forward uses the block's
+ current device instead of forcing a specific device.
"""
diffusion_fn = getattr(self, "_get_diffusion_block_outputs", None)
if getattr(self.model_context, "is_diffusion", False):
+ device = device_override if device_override is not None else self.compress_context.device
return self._get_diffusion_block_outputs(
block,
input_ids,
input_others,
bs,
- self.compress_context.device,
+ device,
self.compress_context.cache_device,
)
diff --git a/auto_round/calibration/diffusion.py b/auto_round/calibration/diffusion.py
index 2476c0f12..fe3ae863f 100644
--- a/auto_round/calibration/diffusion.py
+++ b/auto_round/calibration/diffusion.py
@@ -97,21 +97,40 @@ class DiffusionCalibrator(LLMCalibrator):
if pipe.device != c.model.device:
pipe.to(c.model.device)
+
+ # Check if this is an I2V pipeline (needs calibration image)
+ requires_image = hasattr(c, "_requires_calibration_image") and c._requires_calibration_image()
+
with tqdm(range(1, total + 1), desc="cache block inputs") as pbar:
for ids, prompts in c.dataloader:
if isinstance(prompts, tuple):
prompts = list(prompts)
try:
- pipe(
- prompts,
- guidance_scale=c.guidance_scale,
- num_inference_steps=c.num_inference_steps,
- generator=(
- None
- if c.generator_seed is None
- else torch.Generator(device=pipe.device).manual_seed(c.generator_seed)
- ),
- )
+ if requires_image:
+ # I2V pipeline: pass image and prompt as kwargs
+ image = c._get_calibration_image(len(prompts) if isinstance(prompts, list) else 1)
+ pipe(
+ image,
+ prompt=prompts,
+ guidance_scale=c.guidance_scale,
+ num_inference_steps=c.num_inference_steps,
+ generator=(
+ None
+ if c.generator_seed is None
+ else torch.Generator(device=pipe.device).manual_seed(c.generator_seed)
+ ),
+ )
+ else:
+ pipe(
+ prompts,
+ guidance_scale=c.guidance_scale,
+ num_inference_steps=c.num_inference_steps,
+ generator=(
+ None
+ if c.generator_seed is None
+ else torch.Generator(device=pipe.device).manual_seed(c.generator_seed)
+ ),
+ )
except NotImplementedError:
pass
except Exception as error:
diff --git a/auto_round/calibration/utils.py b/auto_round/calibration/utils.py
index 523561aef..24707564d 100644
--- a/auto_round/calibration/utils.py
+++ b/auto_round/calibration/utils.py
@@ -45,8 +45,13 @@ def _update_inputs(inputs: dict, q_inputs: dict) -> tuple[dict, torch.Tensor]:
model_context = ModelContext()
if model_context.is_diffusion:
- # flux transformer model's blocks will update hidden_states and encoder_hidden_states
input_id_str = [key for key in inputs.keys() if "hidden_state" in key]
+ if input_id_str == ["hidden_states"]:
+ if q_inputs is not None:
+ q_inputs = q_inputs.pop("hidden_states", None)
+ return inputs, q_inputs
+
+ # flux-like diffusion blocks update both hidden_states and encoder_hidden_states
if q_inputs is not None:
q_inputs = {k: q_inputs.pop(k, None) for k in input_id_str}
return inputs, q_inputs
@@ -61,3 +66,16 @@ def _update_inputs(inputs: dict, q_inputs: dict) -> tuple[dict, torch.Tensor]:
if q_inputs is not None:
q_inputs = q_inputs.pop(input_id_str[0], None)
return inputs, q_inputs
+
+
+def _split_inputs_diffusion(inputs: dict) -> tuple[dict, dict]:
+ """Split inputs for diffusion models that only have hidden_states."""
+ input_id_str = [key for key in inputs.keys() if "hidden_state" in key]
+ if input_id_str == ["hidden_states"]:
+ input_ids = inputs.pop("hidden_states", None)
+ input_others = inputs
+ return input_ids, input_others
+
+ input_ids = {k: inputs.pop(k, None) for k in input_id_str}
+ input_others = inputs
+ return input_ids, input_others
diff --git a/auto_round/compressors/data_driven.py b/auto_round/compressors/data_driven.py
index 829408e80..c0dd82e6f 100644
--- a/auto_round/compressors/data_driven.py
+++ b/auto_round/compressors/data_driven.py
@@ -28,6 +28,7 @@ from auto_round import envs
from auto_round.algorithms.alg_config import AlgConfig
from auto_round.calibration.utils import (
_infer_last_cache_name,
+ _split_inputs_diffusion,
_update_inputs,
)
from auto_round.compressors.base import BaseCompressor
@@ -245,6 +246,7 @@ class DataDrivenCompressor(BaseCompressor):
fake_layer = _FakeDecodingLayer()
fake_layer.orig_forward = fake_layer.forward
+ fake_layer._true_orig_forward = lambda *a, **kw: (a, kw)
fake_layer.forward = partial(self._get_block_forward_func(first_block_name), fake_layer)
self.inputs = {}
@@ -437,6 +439,19 @@ class DataDrivenCompressor(BaseCompressor):
input_ids, input_others = self._preprocess_block_inputs(inputs)
+ # For diffusion models, the heuristic split ("hidden_state" in key) may
+ # place keys like encoder_hidden_states in input_ids even though they are
+ # not block outputs. Move those to input_others so they persist across
+ # blocks (only output keys get refreshed via reference_output each iteration).
+ if self.model_context.is_diffusion and isinstance(input_ids, dict):
+ first_block = get_module(model, block_names[0])
+ output_config = self.quantizer.DIFFUSION_OUTPUT_CONFIGS.get(
+ first_block.__class__.__name__, ["hidden_states"]
+ )
+ extra_keys = [k for k in list(input_ids.keys()) if k not in output_config]
+ for k in extra_keys:
+ input_others[k] = input_ids.pop(k)
+
if pbar is None:
pbar = tqdm(range(0, len(block_names), nblocks))
@@ -488,7 +503,7 @@ class DataDrivenCompressor(BaseCompressor):
m = m.to(self.compress_context.device)
card_0_in_high_risk, loss_device = False, self.compress_context.device
- if len(self.compress_context.device_list) > 1:
+ if len(self.compress_context.device_list) > 1 and not self.model_context.is_diffusion:
from accelerate.hooks import AlignDevicesHook, add_hook_to_module
for _n, _mod in m.named_modules():
@@ -500,14 +515,20 @@ class DataDrivenCompressor(BaseCompressor):
bs = self.quantizer.batch_size * self.quantizer.infer_bs_coeff
if q_input is None:
hook_handles = self.quantizer._register_act_max_hook(m)
- reference_output = self.quantizer._get_block_outputs(m, input_ids, input_others, bs)
+ reference_output = self.quantizer._get_block_outputs(
+ m, input_ids, input_others, bs, device_override=loss_device
+ )
for h in hook_handles:
h.remove()
else:
- reference_output = self.quantizer._get_block_outputs(m, input_ids, input_others, bs)
+ reference_output = self.quantizer._get_block_outputs(
+ m, input_ids, input_others, bs, device_override=loss_device
+ )
hook_handles = self.quantizer._register_act_max_hook(m)
if hook_handles:
- self.quantizer._get_block_outputs(m, q_input, input_others, bs, save_output=False)
+ self.quantizer._get_block_outputs(
+ m, q_input, input_others, bs, save_output=False, device_override=loss_device
+ )
for h in hook_handles:
h.remove()
@@ -541,7 +562,7 @@ class DataDrivenCompressor(BaseCompressor):
q_input = None
# ── Infrastructure: hook removal, device cleanup, logging ─────────
- if len(self.compress_context.device_list) > 1:
+ if len(self.compress_context.device_list) > 1 and not self.model_context.is_diffusion:
accelerate.hooks.remove_hook_from_submodules(m)
mv_module_from_gpu(m)
if self.enable_torch_compile:
@@ -652,7 +673,12 @@ class DataDrivenCompressor(BaseCompressor):
logger.info("caching done")
if self.compress_context.low_cpu_mem_usage:
if self.model_context.is_model_patched and not self.compress_context.is_immediate_saving:
- self._offloader(self.model_context.model, all_blocks, clear_memory=True, device_list=self.device_list)
+ self._offloader(
+ self.model_context.model,
+ all_blocks,
+ clear_memory=True,
+ device_list=self.compress_context.device_list,
+ )
if not self._offloader.enabled:
self.compress_context.low_cpu_mem_usage = False
else:
@@ -981,7 +1007,7 @@ class CalibratedRTNCompressor(DataDrivenCompressor):
self.quantizer.batch_size,
self.compress_context.device,
)
- if len(self.compress_context.device_list) > 1:
+ if len(self.compress_context.device_list) > 1 and not self.model_context.is_diffusion:
from accelerate.hooks import AlignDevicesHook, add_hook_to_module
for _, _mod in block.named_modules():
diff --git a/auto_round/compressors/diffusion/README.md b/auto_round/compressors/diffusion/README.md
index 03125e7da..96501c9b6 100644
--- a/auto_round/compressors/diffusion/README.md
+++ b/auto_round/compressors/diffusion/README.md
@@ -60,13 +60,16 @@ auto-round \
For diffusion models, currently we validate quantization on the following models, which involves quantizing the transformer component of the pipeline.
-| Model | calibration dataset | Model Link |
-| --- | --- | --- |
-| black-forest-labs/FLUX.1-dev | COCO2014 | - |
-| Tongyi-MAI/Z-Image | COCO2014 | - |
-| Tongyi-MAI/Z-Image-Turb | COCO2014 | - |
-| stepfun-ai/NextStep-1.1 | COCO2014 | - |
-| AIDC-AI/Ovis-Image-7B | COCO2014 | - |
+| Model | calibration dataset | Model Link |
+|---------------|---------------------|--------------|
+| black-forest-labs/FLUX.1-dev | COCO2014 | - |
+| Tongyi-MAI/Z-Image | COCO2014 | - |
+| Tongyi-MAI/Z-Image-Turb | COCO2014 | - |
+| stepfun-ai/NextStep-1.1 | COCO2014 | - |
+| AIDC-AI/Ovis-Image-7B | COCO2014 | - |
+| Wan-AI/Wan2.2-I2V-A14B-Diffusers | COCO2014 | - |
+| Wan-AI/Wan2.2-TI2V-5B-Diffusers | COCO2014 | - |
+| Wan-AI/Wan2.2-T2V-A14B-Diffusers | COCO2014 | - |
<details>
<summary style="font-size:17px;">Calibration Dataset</summary>
diff --git a/auto_round/compressors/diffusion_mixin.py b/auto_round/compressors/diffusion_mixin.py
index 359b3e0f3..84cde97e3 100644
--- a/auto_round/compressors/diffusion_mixin.py
+++ b/auto_round/compressors/diffusion_mixin.py
@@ -11,12 +11,22 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+import inspect
import os
from typing import Union
import torch
+from tqdm import tqdm
from auto_round.logger import logger
+from auto_round.utils import clear_memory
+from auto_round.utils.device import (
+ dispatch_model_block_wise,
+ dispatch_model_by_all_available_devices,
+ get_major_device,
+ is_auto_device_mapping,
+)
+from auto_round.utils.model import rename_weights_files
class DiffusionMixin:
@@ -49,12 +59,8 @@ class DiffusionMixin:
self.guidance_scale = guidance_scale
self.num_inference_steps = num_inference_steps
self.generator_seed = generator_seed
+ self.pipeline_call_kwargs = dict(kwargs.pop("pipeline_call_kwargs", {}) or {})
- # Mirror old-arch DiffusionCompressor.__init__: when iters > 0, diffusion calibration
- # cannot use batch_size > 1 for non-text modules; fold the extra batch into
- # gradient_accumulate_steps so the effective sample count is unchanged.
- # The authoritative batch_size lives on the AlgConfig (args[0]); kwargs may also
- # carry it from AutoRoundCompatible. Patch BOTH (same pattern as MLLMMixin).
iters = kwargs.get("iters", None)
_alg_cfg = args[0] if args else None
if iters is None and _alg_cfg is not None:
@@ -98,12 +104,6 @@ class DiffusionMixin:
# Call parent class __init__ (will be DataDrivenCompressor, ImatrixCompressor, etc)
super().__init__(*args, **kwargs)
- # Mirror old-arch DiffusionCompressor._align_device_and_dtype: unconditionally
- # cast the full diffusion pipeline (VAE, text encoder, etc.) to the transformer's
- # dtype so that calibration's pipe(...) call doesn't crash with dtype mismatches
- # when the transformer is force-cast to bf16 for activation quantization.
- # Note: pipe.dtype only reflects the primary component, so an equality check would
- # miss mixed-dtype pipelines where e.g. the VAE is still float32.
pipe = getattr(self.model_context, "pipe", None)
model = getattr(self.model_context, "model", None)
if pipe is not None and model is not None:
@@ -121,40 +121,504 @@ class DiffusionMixin:
"""
return "diffusion"
- def save_quantized(self, output_dir=None, format="auto_round", inplace=True, **kwargs):
+ def _build_pipeline_call_kwargs(self, pipe, prompts):
+ """Build kwargs for pipeline.__call__."""
+ call_kwargs = {
+ "guidance_scale": self.guidance_scale,
+ "num_inference_steps": self.num_inference_steps,
+ "generator": (
+ None
+ if self.generator_seed is None
+ else torch.Generator(device=pipe.device).manual_seed(self.generator_seed)
+ ),
+ }
+ call_kwargs.update(self.pipeline_call_kwargs)
+
+ if self._requires_calibration_image():
+ call_kwargs.setdefault(
+ "image", self._get_calibration_image(len(prompts) if isinstance(prompts, list) else 1)
+ )
+ call_kwargs.setdefault("prompt", prompts)
+
+ return call_kwargs
+
+ def _requires_calibration_image(self) -> bool:
+ """Return True when the pipeline's __call__ has a required 'image' parameter.
+
+ I2V (image-to-video) pipelines like WanImageToVideoPipeline require a PIL/torch
+ image as input. This is detected by checking whether 'image' is a positional-or-
+ keyword parameter without a default value.
+ """
+ image_param = inspect.signature(self.model_context.pipe.__call__).parameters.get("image")
+ return image_param is not None and image_param.default is inspect.Parameter.empty
+
+ def _get_calibration_image(self, batch_size: int):
+ """Return a synthetic PIL Image for I2V pipeline calibration."""
+ params = inspect.signature(self.model_context.pipe.__call__).parameters
+ width_param = params.get("width")
+ height_param = params.get("height")
+ width = (
+ 832
+ if width_param is None or width_param.default in (inspect.Parameter.empty, None)
+ else width_param.default
+ )
+ height = (
+ 480
+ if height_param is None or height_param.default in (inspect.Parameter.empty, None)
+ else height_param.default
+ )
+ from PIL import Image # pylint: disable=E0401
+
+ image = Image.new("RGB", (int(width), int(height)), color=(127, 127, 127))
+ if batch_size == 1:
+ return image
+ return [image.copy() for _ in range(batch_size)]
+
+ def _find_additional_transformers(self):
+ """Find transformer components beyond the primary one (e.g. transformer_2 in WAN)."""
+ pipe = getattr(self.model_context, "pipe", None)
+ if pipe is None:
+ return []
+ result = []
+ for comp_name in pipe.components:
+ comp = getattr(pipe, comp_name, None)
+ if (
+ comp_name.startswith("transformer")
+ and comp_name != "transformer"
+ and comp is not None
+ and isinstance(comp, torch.nn.Module)
+ ):
+ result.append((comp_name, comp))
+ return result
+
+ def _align_device_and_dtype_for_secondary(self, transformer_name: str):
+ """Align dtype and dispatch secondary transformer for multi-transformer pipelines."""
+ pipe = getattr(self.model_context, "pipe", None)
+ model = getattr(self.model_context, "model", None)
+ if pipe is None or model is None:
+ return
+
+ # Cast full pipeline to transformer's dtype
+ is_nextstep = hasattr(model, "config") and getattr(model.config, "model_type", None) == "nextstep"
+ if not is_nextstep:
+ pipe.to(model.dtype)
+
+ # Dispatch secondary transformer to GPU(s)
+ device_map = getattr(self.compress_context, "device_map", None)
+ device_list = getattr(self.compress_context, "device_list", [])
+ multi_device = is_auto_device_mapping(device_map) and len(device_list) > 1
+
+ if multi_device:
+ comp_device = device_list[-1]
+ for comp_name in pipe.components:
+ comp = getattr(pipe, comp_name, None)
+ if comp is None or comp is model or not hasattr(comp, "to"):
+ continue
+ is_other_transformer = (
+ comp_name.startswith("transformer")
+ and isinstance(comp, torch.nn.Module)
+ and next(comp.parameters()).device.type == "cpu"
+ )
+ is_other_component = not comp_name.startswith("transformer")
+ if is_other_transformer or is_other_component:
+ if isinstance(comp, torch.nn.Module) and hasattr(comp, "dtype") and comp.dtype != model.dtype:
+ comp.to(dtype=model.dtype)
+ try:
+ comp.to(comp_device)
+ except (NotImplementedError, RuntimeError):
+ continue
+
+ self.model_context.model = dispatch_model_block_wise(model, device_map)
+ setattr(pipe, transformer_name, self.model_context.model)
+ elif device_map is not None:
+ target_device = get_major_device(device_map)
+ pipe.to(target_device)
+
+ @torch.no_grad()
+ def calib(self, nsamples, bs):
+ """Perform diffusion-specific calibration for quantization.
+
+ Override parent's calib method to use diffusion dataset loading logic.
+ The diffusion pipeline is read from ``self.model_context.pipe``.
+ """
+ from auto_round.compressors.diffusion.dataset import get_diffusion_dataloader
+
+ pipe = self.model_context.pipe
+ if pipe is None:
+ raise ValueError(
+ "Diffusion pipeline not found in model_context. " "Ensure the model was loaded as a diffusion model."
+ )
+
+ logger.warning(
+ "Diffusion model will catch nsamples * num_inference_steps inputs, "
+ "you can reduce nsamples or num_inference_steps if OOM or take too much time."
+ )
+ if isinstance(self.dataset, str):
+ dataset = self.dataset.replace(" ", "")
+ self.dataloader, self.batch_size, self.gradient_accumulate_steps = get_diffusion_dataloader(
+ dataset=dataset,
+ bs=self.batch_size,
+ seed=self.seed,
+ nsamples=self.nsamples,
+ gradient_accumulate_steps=self.gradient_accumulate_steps,
+ )
+ else:
+ self.dataloader = self.dataset
+ total_cnt = 0
+
+ total = nsamples if not hasattr(self.dataloader, "len") else min(nsamples, len(self.dataloader))
+
+ # NOTE: we intentionally skip the sequential offloading check here (the guard that
+ # exits when pipe is already dispatched). In new-arch diffusion, the pipe may
+ # already be dispatched to multi-device in a prior calib call. The dispatch
+ # state is preserved across calls, so re-dispatching or moving is unnecessary and
+ # would break the existing placement.
+ if (
+ hasattr(self.model, "hf_device_map")
+ and len(self.model.hf_device_map) > 1
+ and pipe.device != self.model.device
+ and torch.device(self.model.device).type in ["cuda", "xpu"]
+ ):
+ logger.warning(
+ "Diffusion model is activated sequential model offloading. "
+ "Pipe may already be dispatched from a prior calib call. "
+ "Skipping re-dispatch to avoid breaking the existing placement."
+ )
+
+ if pipe.device != self.model.device:
+ pipe.to(self.model.device)
+
+ device_map = getattr(self.compress_context, "device_map", None)
+ device_list = getattr(self.compress_context, "device_list", [])
+ # Skip dispatch for secondary transformers
+ if (
+ not getattr(self, "_inputs_cached", False)
+ and device_map is not None
+ and is_auto_device_mapping(device_map)
+ and len(device_list) > 1
+ ):
+ pipe_transformer = getattr(pipe, "transformer", None)
+ if self.model_context.model is not pipe_transformer:
+ pass # secondary transformer — skip pipeline dispatch
+ else:
+ pipe = dispatch_model_by_all_available_devices(pipe, device_map)
+
+ with tqdm(range(1, total + 1), desc="cache block inputs") as pbar:
+ for ids, prompts in self.dataloader:
+ if isinstance(prompts, tuple):
+ prompts = list(prompts)
+ pipe_kwargs = self._build_pipeline_call_kwargs(pipe, prompts)
+ try:
+ if self._requires_calibration_image() or "prompt" in pipe_kwargs:
+ # I2V pipeline: 'image' is the first positional arg, so pass
+ # 'prompt' as keyword to avoid "multiple values for argument 'image'".
+ pipe(**pipe_kwargs)
+ else:
+ pipe(prompts, **pipe_kwargs)
+ except NotImplementedError:
+ pass
+ except Exception as error:
+ raise error
+ step = len(prompts)
+ total_cnt += step
+ pbar.update(step)
+ if total_cnt >= nsamples:
+ break
+ if total_cnt == 0:
+ logger.error(
+ f"no data has been cached, please provide more data with sequence length >={self.seqlen} in the "
+ f"dataset or decease the sequence length"
+ )
+ exit(-1)
+ elif total_cnt < nsamples:
+ logger.warning(
+ f"Insufficient number of samples collected may affect the quantization. "
+ f"target samples count is {nsamples}, while valid samples count is {total_cnt}"
+ )
+ if total_cnt < self.batch_size:
+ raise ValueError(
+ f"valid samples is less than batch_size({self.batch_size}),"
+ " please adjust self.batch_size or seqlen."
+ )
+ max_len = (total_cnt // self.batch_size) * self.batch_size
+ for k, v in self.inputs.items():
+ for key in v:
+ if isinstance(v[key], list) and len(v[key]) == total_cnt:
+ self.inputs[k][key] = v[key][:max_len]
+
+ # torch.cuda.empty_cache()
+
+ def try_cache_inter_data_gpucpu(self, *args, **kwargs):
+ """Skip re-caching when DiffusionMixin.quantize has already populated self.inputs.
+
+ CalibCompressor.quantize() always calls try_cache_inter_data_gpucpu, but for
+ diffusion models the inputs were already collected by the diffusion pipeline
+ in DiffusionMixin.quantize(). Return the cached data directly.
+ """
+ if getattr(self, "_inputs_cached", False):
+ self._inputs_cached = False
+ return self.inputs
+ if hasattr(super(), "try_cache_inter_data_gpucpu"):
+ return super().try_cache_inter_data_gpucpu(*args, **kwargs)
+
+ def quantize(self):
+ """Quantize the diffusion model.
+
+ Overrides the parent to use diffusion-specific cache_inter_data instead of
+ the LLM-specific calib path. The diffusion pipeline forward is used to collect
+ block inputs (via _replace_forward hooks), then those inputs are passed to the
+ standard CalibCompressor quantization loop.
+
+ For dual-transformer pipelines (e.g. WAN with transformer + transformer_2),
+ this method quantizes all transformers sequentially.
+ """
+ from auto_round.utils import find_matching_blocks, get_block_names
+ from auto_round.utils.common import flatten_list
+
+ self.post_init()
+
+ # Get block names and call cache_inter_data to populate self.inputs
+ if bool(self.quantizer.quant_block_list):
+ all_blocks = self.quantizer.quant_block_list
+ else:
+ all_blocks = get_block_names(self.model_context.model)
+ if len(all_blocks) == 0:
+ logger.warning("could not find blocks, exit with original model")
+ return self.model_context.model, self.quantizer.layer_config
+
+ if not self.has_variable_block_shape:
+ to_cache_block_names = [block[0] for block in all_blocks]
+ else:
+ to_cache_block_names = flatten_list(all_blocks)
+
+ # Check for additional transformers (e.g. transformer_2 in WAN)
+ additional = self._find_additional_transformers()
+ if not additional:
+ # Single-transformer path: let calib() own pipeline dispatch.
+ pipe = self.model_context.pipe
+ device_map = getattr(self.compress_context, "device_map", None)
+ if device_map is not None and not is_auto_device_mapping(device_map) and not isinstance(device_map, int):
+ target_device = get_major_device(device_map)
+ # Skip if the transformer is already on the target device to avoid
+ # redundant full-model transfer that exhausts GPU memory.
+ transformer = getattr(pipe, "transformer", None) or getattr(pipe, "unet", None)
+ param = next(transformer.parameters(), None) if transformer else None
+ skip_move = param is not None and get_major_device(str(param.device)) == target_device
+ if not skip_move:
+ pipe.to(target_device)
+
+ logger.info("start to cache block inputs")
+ all_inputs = self.try_cache_inter_data_gpucpu(
+ to_cache_block_names,
+ self.nsamples,
+ layer_names=[],
+ )
+ self.inputs = all_inputs
+ clear_memory(device_list=self.compress_context.device_list)
+ self._inputs_cached = True
+ return super().quantize()
+
+ # Dual-transformer path: quantize all transformers sequentially
+ logger.info("Detected multi-transformer diffusion pipeline, quantizing all transformers")
+
+ # Ensure at least 2 inference steps so both transformers are exercised during calibration
+ orig_steps = getattr(self, "num_inference_steps", None) or 50
+ if orig_steps < 2:
+ logger.warning(
+ f"num_inference_steps={orig_steps} is too low for dual-transformer "
+ f"quantization — increasing to 2 so all transformers receive calibration data."
+ )
+ self.num_inference_steps = 2
+
+ # Disable low_cpu_mem_usage so quantized models stay in memory during multi-transformer
+ # quantization.
+ orig_low_cpu = self.compress_context.low_cpu_mem_usage
+ orig_immediate_saving = self.compress_context.is_immediate_saving
+ self.compress_context.low_cpu_mem_usage = False
+ # Defer shard writing until all transformers are quantized. Immediate saving
+ # offloads the primary transformer to meta during the first pass, which breaks
+ # later calibration when the pipeline switches to transformer_2.
+ self.compress_context.is_immediate_saving = False
+
+ # Store primary transformer state
+ primary_model = self.model_context.model
+ primary_layer_config = dict(self.layer_config) if self.layer_config else {}
+ primary_quant_block_list = list(self.quant_block_list) if self.quant_block_list else []
+ quantized_extras = {}
+
+ # Quantize primary transformer
+ logger.info("start to cache block inputs for primary transformer")
+ all_inputs = self.try_cache_inter_data_gpucpu(
+ to_cache_block_names,
+ self.nsamples,
+ layer_names=[],
+ )
+ self.inputs = all_inputs
+ clear_memory(device_list=self.compress_context.device_list)
+ self._inputs_cached = True
+ super().quantize()
+
+ # Clear stale hf_device_map so cache_inter_data can re-dispatch for next transformer
+ if hasattr(primary_model, "hf_device_map"):
+ delattr(primary_model, "hf_device_map")
+
+ # Quantize additional transformers
+ for comp_name, transformer in additional:
+ logger.info(f"Quantizing {comp_name}")
+
+ # Reset quantization state for new transformer
+ self.model_context.model = transformer
+ self.model_context.quantized = False
+ self._post_init_done = False
+
+ # Re-align device/dtype and dispatch pipeline for secondary transformer
+ self._align_device_and_dtype_for_secondary(comp_name)
+
+ # Re-run post_init to set up quantizer for new model
+ self.post_init()
+
+ # Get block names for new transformer
+ all_blocks = get_block_names(self.model_context.model)
+ self.quant_block_list = find_matching_blocks(self.model_context.model, all_blocks, None)
+ self.layer_config = {}
+
+ # Get new block names for caching
+ if bool(self.quantizer.quant_block_list):
+ all_blocks = self.quantizer.quant_block_list
+ else:
+ all_blocks = get_block_names(self.model_context.model)
+ if len(all_blocks) == 0:
+ logger.warning(f"could not find blocks in {comp_name}, skipping")
+ continue
+
+ if not self.has_variable_block_shape:
+ to_cache_block_names = [block[0] for block in all_blocks]
+ else:
+ to_cache_block_names = flatten_list(all_blocks)
+
+ logger.info(f"start to cache block inputs for {comp_name}")
+ all_inputs = self.try_cache_inter_data_gpucpu(
+ to_cache_block_names,
+ self.nsamples,
+ layer_names=[],
+ )
+ self.inputs = all_inputs
+ clear_memory(device_list=self.compress_context.device_list)
+ self._inputs_cached = True
+ super().quantize()
+
+ # Store quantized transformer
+ quantized_extras[comp_name] = (self.model_context.model, dict(self.layer_config))
+ # Also update the pipeline to reference the quantized transformer
+ setattr(self.model_context.pipe, comp_name, self.model_context.model)
+
+ # Restore primary transformer state
+ self.model_context.model = primary_model
+ self.model_context.quantized = True
+ self.layer_config = primary_layer_config
+ self.quant_block_list = primary_quant_block_list
+ self._quantized_transformers = quantized_extras
+ self.compress_context.low_cpu_mem_usage = orig_low_cpu
+ self.compress_context.is_immediate_saving = orig_immediate_saving
+ self.num_inference_steps = orig_steps
+
+ return self.model_context.model, self.quantizer.layer_config
+
+ def save_quantized(self, output_dir=None, format="auto_round", inplace=True, return_folders=False, **kwargs):
"""Save the quantized model to the specified output directory in the specified format.
+ For multi-transformer pipelines, all quantized transformers are saved.
+
Args:
output_dir (str, optional): The directory to save the quantized model. Defaults to None.
format (str, optional): The format in which to save the model. Defaults to "auto_round".
inplace (bool, optional): Whether to modify the model in place. Defaults to True.
+ return_folders (bool, optional): Whether to return the save folder paths. Defaults to False.
**kwargs: Additional keyword arguments specific to the export format.
Returns:
- object: The compressed model object.
+ object: The compressed model object, or (compressed_model, folders) if return_folders is True.
"""
if output_dir is None:
- return super().save_quantized(output_dir, format=format, inplace=inplace, **kwargs)
+ return super().save_quantized(
+ output_dir, format=format, inplace=inplace, return_folders=return_folders, **kwargs
+ )
pipe = self.model_context.pipe
+ quantized_transformers = getattr(self, "_quantized_transformers", {})
compressed_model = None
+ folders = []
+ has_multiple_quantized_transformers = bool(quantized_transformers)
+
+ # Handle multi-format (convert string to list if needed)
+ _format = format
+ if isinstance(_format, str):
+ from auto_round.formats import get_formats
+
+ _format = get_formats(_format, self)
+
for name in pipe.components.keys():
val = getattr(pipe, name)
sub_module_path = (
os.path.join(output_dir, name) if os.path.basename(os.path.normpath(output_dir)) != name else output_dir
)
- if (
- hasattr(val, "config")
- and hasattr(val.config, "_name_or_path")
- and val.config._name_or_path == self.model.config._name_or_path
- ):
+ target_output_dir = (
+ sub_module_path
+ if has_multiple_quantized_transformers or not self.compress_context.is_immediate_saving
+ else output_dir
+ )
+ if name in quantized_transformers:
+ # Save secondary quantized transformer
+ saved_model = self.model_context.model
+ saved_lc = self.layer_config
+ saved_immediate_saving = self.compress_context.is_immediate_saving
+ self.model_context.model, self.layer_config = quantized_transformers[name]
+ saved_subfolder = getattr(self.model_context.model, "_autoround_pipeline_subfolder", None)
+ if has_multiple_quantized_transformers:
+ self.compress_context.is_immediate_saving = False
+ if saved_subfolder is not None:
+ delattr(self.model_context.model, "_autoround_pipeline_subfolder")
compressed_model = super().save_quantized(
- output_dir=sub_module_path if not self.compress_context.is_immediate_saving else output_dir,
- format=format,
+ output_dir=target_output_dir,
+ format=_format,
inplace=inplace,
+ return_folders=False,
**kwargs,
)
+ self.compress_context.is_immediate_saving = saved_immediate_saving
+ if saved_subfolder is not None:
+ self.model_context.model._autoround_pipeline_subfolder = saved_subfolder
+ self.model_context.model = saved_model
+ self.layer_config = saved_lc
+ elif val is self.model_context.model:
+ # Save primary quantized transformer
+ saved_immediate_saving = self.compress_context.is_immediate_saving
+ saved_subfolder = getattr(self.model_context.model, "_autoround_pipeline_subfolder", None)
+ if has_multiple_quantized_transformers:
+ self.compress_context.is_immediate_saving = False
+ if saved_subfolder is not None:
+ delattr(self.model_context.model, "_autoround_pipeline_subfolder")
+ compressed_model = super().save_quantized(
+ output_dir=target_output_dir,
+ format=_format,
+ inplace=inplace,
+ return_folders=False,
+ **kwargs,
+ )
+ self.compress_context.is_immediate_saving = saved_immediate_saving
+ if saved_subfolder is not None:
+ self.model_context.model._autoround_pipeline_subfolder = saved_subfolder
elif val is not None and hasattr(val, "save_pretrained"):
val.save_pretrained(sub_module_path)
- pipe.config.save_pretrained(output_dir)
+ continue
+
+ if name.startswith("transformer"):
+ rename_weights_files(target_output_dir)
+
+ folders.append(target_output_dir)
+
+ pipe.save_config(output_dir)
+
+ if return_folders:
+ return compressed_model, folders
return compressed_model
diff --git a/auto_round/context/model.py b/auto_round/context/model.py
index a29db4073..2cd5273d1 100644
--- a/auto_round/context/model.py
+++ b/auto_round/context/model.py
@@ -76,6 +76,9 @@ class ModelContext(BaseContext):
self.is_diffusion = False
self.is_model_patched = False
self.is_moe_model = False
+ # Set by CalibCompressor._replace_forward; used by recover_forward to detect
+ # new-arch diffusion mode where positional wrapper must be stripped after caching.
+ self._has_true_orig_forward_set = False
assert model is not None, "model must be provided for ModelContext"
self.model = model
@@ -282,14 +285,44 @@ class ModelContext(BaseContext):
self.hook_handles = hook_handles
- def recover_forward(self):
- """Recovers the forward function."""
+ def recover_forward(self, restore_positional_wrapper=None):
+ """Recovers the forward function.
+
+ Args:
+ restore_positional_wrapper: If True, restores forward to the wrapped version
+ (needed for LLM calibration where positional wrapper is used during quantization).
+ If False, restores to the true original forward (needed for diffusion).
+ If None (default), auto-detects: uses False for diffusion models.
+ """
assert self._init_model, "should load and initialize model first"
+ # Auto-detect for diffusion: when _true_orig_forward is present (set by
+ # CalibCompressor._replace_forward), we are in new-arch diffusion mode where
+ # the positional wrapper must be fully removed after caching.
+ if restore_positional_wrapper is None:
+ restore_positional_wrapper = not getattr(self, "_has_true_orig_forward_set", False)
+ if not restore_positional_wrapper:
+ logger.debug("recover_forward: auto-detected diffusion mode, stripping positional wrapper")
+
for n, m in self.model.named_modules():
if hasattr(m, "orig_forward"):
- m.forward = m.orig_forward
- delattr(m, "orig_forward")
+ true_orig = getattr(m, "_true_orig_forward", m.orig_forward)
+ if restore_positional_wrapper:
+ # Restore orig_forward so that any wrapper (e.g. from
+ # wrap_block_forward_positional_to_kwargs) can still access it.
+ # The wrapper holds a closure reference to orig_forward.
+ m.forward = getattr(m, "_wrapped_forward_before_replace", m.orig_forward)
+ m.orig_forward = true_orig
+ else:
+ # Full recovery: restore the true original forward. Used for diffusion
+ # where the positional wrapper must be fully removed after caching.
+ m.forward = true_orig
+ # Keep _true_orig_forward so the wrapped forward's base_hook can
+ # still call it during quantization tuning.
+ m._true_orig_forward = true_orig
+ delattr(m, "orig_forward")
+ if hasattr(m, "_wrapped_forward_before_replace"):
+ delattr(m, "_wrapped_forward_before_replace")
for hook_handle in self.hook_handles:
hook_handle.remove()
self.hook_handles = []
diff --git a/auto_round/eval/evaluation.py b/auto_round/eval/evaluation.py
index 5933bbc0d..2288dcafc 100644
--- a/auto_round/eval/evaluation.py
+++ b/auto_round/eval/evaluation.py
@@ -415,6 +415,10 @@ def run_model_evaluation(model, tokenizer, autoround, folders, formats, device_s
# Handle diffusion models separately
if getattr(autoround, "diffusion", False):
+ prompt = getattr(args, "prompt", None)
+ prompt_file = getattr(args, "prompt_file", None)
+ if prompt is None and prompt_file is None:
+ return
evaluate_diffusion_model(args, autoround=autoround, model=model)
return
diff --git a/auto_round/export/utils.py b/auto_round/export/utils.py
index 837a9f073..85765d11b 100644
--- a/auto_round/export/utils.py
+++ b/auto_round/export/utils.py
@@ -75,6 +75,18 @@ def _resolve_pipeline_source_dir(model: nn.Module) -> str | None:
return None
+def _resolve_model_source_dir(model: nn.Module) -> str | None:
+ candidates = [
+ getattr(model, "name_or_path", None),
+ getattr(getattr(model, "config", None), "_name_or_path", None),
+ getattr(getattr(model, "config", None), "name_or_path", None),
+ ]
+ for candidate in candidates:
+ if isinstance(candidate, str) and candidate:
+ return candidate
+ return None
+
+
def _copy_pipeline_artifact(model_dir: str, relative_path: str, output_dir: str) -> None:
target_path = os.path.join(output_dir, relative_path)
os.makedirs(os.path.dirname(target_path), exist_ok=True)
@@ -212,16 +224,14 @@ def save_model(
else:
model.save_pretrained(save_dir, max_shard_size=max_shard_size, safe_serialization=safe_serialization)
+ source_dir = _resolve_model_source_dir(model)
+
# Allow disabling copy_missing_tensors_from_source via env var AR_DISABLE_COPY_MTP_WEIGHTS, default enabled
if not envs.AR_DISABLE_COPY_MTP_WEIGHTS:
try:
- if (
- hasattr(model, "config")
- and hasattr(model.config, "_name_or_path")
- and model.config.name_or_path is not None # set None for tiny model
- ):
+ if source_dir is not None:
copy_missing_tensors_from_source(
- source_dir=model.config._name_or_path,
+ source_dir=source_dir,
target_dir=save_dir,
)
except Exception as e:
@@ -245,11 +255,7 @@ def save_model(
json.dump(model.config.quantization_config, f, indent=2)
try:
- if (
- hasattr(model, "config")
- and hasattr(model.config, "_name_or_path")
- and model.config.name_or_path is not None # set None for tiny model
- ):
+ if source_dir is not None:
copy_python_files_from_model_cache(model, save_dir)
except Exception as e:
logger.warning("Skipping source model Python file copy due to error: %s", e)
diff --git a/auto_round/modeling/fused_moe/moe_experts_interface.py b/auto_round/modeling/fused_moe/moe_experts_interface.py
index d8fc9ab46..854d9e186 100644
--- a/auto_round/modeling/fused_moe/moe_experts_interface.py
+++ b/auto_round/modeling/fused_moe/moe_experts_interface.py
@@ -321,11 +321,16 @@ def _detect_expert_projections(module: nn.Module) -> dict[str, dict]:
if param is not None and isinstance(param, nn.Parameter) and param.dim() == 3:
detected[proj_name] = config
- # If no known patterns found, scan for any 3D Parameter (future-proofing)
+ # If no known patterns found, scan for any 3D Parameter (future-proofing),
+ # but exclude known non-expert 3D parameters that exist on some transformer
+ # architectures (e.g. scale_shift_table on WanTransformer3DModel).
+ _NON_EXPERT_3D_PARAMS = frozenset(["scale_shift_table"])
if not detected:
for attr_name in dir(module):
if attr_name.startswith("_"):
continue
+ if attr_name in _NON_EXPERT_3D_PARAMS:
+ continue
param = getattr(module, attr_name, None)
if param is not None and isinstance(param, nn.Parameter) and param.dim() == 3:
# Use default config for unknown projections
diff --git a/auto_round/special_model_handler.py b/auto_round/special_model_handler.py
index 6f11b454c..68ac5b540 100644
--- a/auto_round/special_model_handler.py
+++ b/auto_round/special_model_handler.py
@@ -58,6 +58,7 @@ SPECIAL_SHARED_CACHE_KEYS = {
"Gemma3ForConditionalGeneration": ("position_embeddings_global", "position_embeddings_local")
}
SPECIAL_SHARED_CACHE_KEYS["MiniMaxText01ForCausalLM"] = ("slope_rate",)
+SPECIAL_SHARED_CACHE_KEYS["WanTransformer3DModel"] = ("rotary_emb",)
MISTRAL_3_2_MODELS = ["Mistral-Small-3.2", "Magistral-Small", "Devstral-Small"]
diff --git a/auto_round/utils/device.py b/auto_round/utils/device.py
index e90d78431..f864a941e 100644
--- a/auto_round/utils/device.py
+++ b/auto_round/utils/device.py
@@ -1489,7 +1489,8 @@ def dispatch_model_block_wise(model: torch.nn.Module, device_map: str, max_mem_r
max_memory=new_max_memory,
no_split_module_classes=no_split_modules,
)
- model.tie_weights()
+ if hasattr(model, "tie_weights"):
+ model.tie_weights()
device_map = infer_auto_device_map(model, max_memory=new_max_memory, no_split_module_classes=no_split_modules)
if len(devices) > 1 and "cpu" in device_map.values():
logger.warning(
@@ -1918,20 +1919,83 @@ def dispatch_model_by_all_available_devices(
# move the entire pipeline to the (first) device.
pipe.to(devices[0] if devices else "cuda:0")
return pipe
- # Multi-device path: recursively dispatch the main sub-model,
- # then move all remaining pipeline components to the primary device.
+ # Multi-device path: dispatch the main sub-model across devices,
+ # reserving memory on the primary device for non-target components
+ # (text encoder, VAE, etc.) to avoid OOM.
main_model = getattr(pipe, main_attr)
- dispatched = dispatch_model_by_all_available_devices(main_model, _device_map)
- setattr(pipe, main_attr, dispatched)
primary_device = devices[0]
+ # Place non-main components on the last device
+ comp_device = devices[-1]
for attr, component in pipe.components.items():
if attr == main_attr:
continue
- if isinstance(component, torch.nn.Module):
+ if not isinstance(component, torch.nn.Module):
+ continue
+ # Align dtype on CPU first to avoid wasting GPU memory
+ if hasattr(component, "dtype") and component.dtype != main_model.dtype:
try:
- component.to(primary_device)
+ component.to(dtype=main_model.dtype)
except Exception:
pass
+ try:
+ component.to(comp_device)
+ except (NotImplementedError, RuntimeError):
+ pass
+
+ from auto_round.utils.common import normalize_no_split_modules
+
+ no_split_modules = normalize_no_split_modules(getattr(main_model, "_no_split_modules", []))
+
+ # dispatch_model_block_wise queries free memory after non-main
+ # components are already placed, so the budget naturally accounts for
+ # them. Use the same approach here.
+ dispatched = dispatch_model_block_wise(main_model, device_map)
+ setattr(pipe, main_attr, dispatched)
+
+ # Install manual pre/post hooks to move tensors.
+ unique_devices = set()
+ if hasattr(dispatched, "hf_device_map"):
+ unique_devices = {v for v in dispatched.hf_device_map.values() if v not in ("cpu", "disk")}
+ if len(unique_devices) <= 1:
+ execution_device = primary_device
+ try:
+ execution_device = next(dispatched.parameters()).device
+ except StopIteration:
+ execution_device = torch.device(primary_device)
+
+ if hasattr(dispatched, "_hf_hook") and hasattr(dispatched._hf_hook, "execution_device"):
+ dispatched._hf_hook.execution_device = execution_device
+
+ if not getattr(dispatched, "_autoround_align_inputs_hook_installed", False):
+ _first_param_device = execution_device
+ _pipeline_device = torch.device(comp_device)
+
+ def _align_all_inputs_pre_hook(module, args, kwargs):
+ try:
+ target = next(module.parameters()).device
+ except StopIteration:
+ target = _first_param_device
+ new_args = tuple(a.to(target) if isinstance(a, torch.Tensor) else a for a in args)
+ new_kwargs = {k: v.to(target) if isinstance(v, torch.Tensor) else v for k, v in kwargs.items()}
+ return new_args, new_kwargs
+
+ def _move_outputs_back_hook(module, input, output):
+ def _to_device(obj, device):
+ if isinstance(obj, torch.Tensor):
+ return obj.to(device) if obj.device != device else obj
+ if isinstance(obj, (tuple, list)):
+ converted = [_to_device(o, device) for o in obj]
+ return type(obj)(converted)
+ if isinstance(obj, dict):
+ return {k: _to_device(v, device) for k, v in obj.items()}
+ return obj
+
+ return _to_device(output, _pipeline_device)
+
+ dispatched.register_forward_pre_hook(_align_all_inputs_pre_hook, with_kwargs=True)
+ dispatched.register_forward_hook(_move_outputs_back_hook)
+ dispatched._autoround_align_inputs_hook_installed = True
+
return pipe
if device_map is None:
diff --git a/auto_round/utils/model.py b/auto_round/utils/model.py
index db7f2294f..8569fab9c 100644
--- a/auto_round/utils/model.py
+++ b/auto_round/utils/model.py
@@ -732,7 +732,7 @@ def diffusion_load_model(
component_config = json.load(file)
torch_dtype[k] = component_config.get("torch_dtype", "auto")
- pipe = pipelines.auto_pipeline.AutoPipelineForText2Image.from_pretrained(
+ pipe = pipelines.pipeline_utils.DiffusionPipeline.from_pretrained(
pretrained_model_name_or_path, torch_dtype=torch_dtype
)
pipe_config = pipe.load_config(pretrained_model_name_or_path)
@@ -767,6 +767,20 @@ def diffusion_load_model(
# non-meta model uses model.save_pretrained for model and config saving
setattr(model, "save_pretrained", partial(model_save_pretrained, model))
+
+ for comp_name in pipe.components:
+ comp = getattr(pipe, comp_name, None)
+ if (
+ comp_name.startswith("transformer")
+ and comp_name != "transformer"
+ and comp is not None
+ and isinstance(comp, torch.nn.Module)
+ ):
+ setattr(
+ comp.config, "save_pretrained", partial(config_save_pretrained, comp.config, "config.json", model=comp)
+ )
+ setattr(comp, "save_pretrained", partial(model_save_pretrained, comp))
+
return pipe, model.to(device)
@@ -2000,14 +2014,18 @@ def wrap_block_forward_positional_to_kwargs(base_hook):
positional args to keyword args, all inputs are properly accumulated
across calibration samples.
"""
- _param_names = None
+ _param_names_cache: dict = {}
def forward(m, hidden_states=None, *positional_inputs, **kwargs):
- nonlocal _param_names
if positional_inputs:
- if _param_names is None:
- sig = inspect.signature(m.orig_forward)
- _param_names = [p for p in sig.parameters.keys() if p != "self"]
+ m_id = id(m)
+ if m_id not in _param_names_cache:
+ # Prefer _true_orig_forward (set by new-arch CalibCompressor._replace_forward)
+ # over orig_forward (which points to the wrapped forward after wrapping).
+ sig_target = getattr(m, "_true_orig_forward", None) or m.orig_forward
+ sig = inspect.signature(sig_target)
+ _param_names_cache[m_id] = [p for p in sig.parameters.keys() if p != "self"]
+ _param_names = _param_names_cache[m_id]
for i, val in enumerate(positional_inputs):
param_idx = i + 1 # hidden_states is params[0]
if param_idx < len(_param_names):