PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Feats: Quantize/save/evaluate the Wan-AI/WAN2.2 models in w4a16 format (#1678)
Liang Lvf1db02a3f50d
diff --git a/auto_round/algorithms/quantization/base.py b/auto_round/algorithms/quantization/base.pyindex fd39f547e..b0e0cbf6b 100644--- a/auto_round/algorithms/quantization/base.py+++ b/auto_round/algorithms/quantization/base.py@@ -53,6 +53,7 @@ class BaseQuantizers:"FluxSingleTransformerBlock": ["encoder_hidden_states", "hidden_states"],"OvisImageTransformerBlock": ["encoder_hidden_states", "hidden_states"],"OvisImageSingleTransformerBlock": ["encoder_hidden_states", "hidden_states"],+ "WanTransformerBlock": ["hidden_states"],}def __init__(self, config: QuantizationConfig):@@ -368,21 +369,28 @@ class BaseQuantizers:input_others,bs: int,save_output: bool = True,+ device_override: Union[torch.device, str, None] = None,):"""Compute the output of a block for calibration inputs.Shared by SignRoundQuantizer and OptimizedRTNQuantizer. Algorithm-specificblock-forward selection (compile vs. plain) is handled here based on``enable_alg_ext`` and act-quantization flags.++ Args:+ device_override: Override the target device. Used by diffusion with+ multi-device dispatch to pass None so block_forward uses the block's+ current device instead of forcing a specific device."""diffusion_fn = getattr(self, "_get_diffusion_block_outputs", None)if getattr(self.model_context, "is_diffusion", False):+ device = device_override if device_override is not None else self.compress_context.devicereturn self._get_diffusion_block_outputs(block,input_ids,input_others,bs,- self.compress_context.device,+ device,self.compress_context.cache_device,)diff --git a/auto_round/calibration/diffusion.py b/auto_round/calibration/diffusion.pyindex 2476c0f12..fe3ae863f 100644--- a/auto_round/calibration/diffusion.py+++ b/auto_round/calibration/diffusion.py@@ -97,21 +97,40 @@ class DiffusionCalibrator(LLMCalibrator):if pipe.device != c.model.device:pipe.to(c.model.device)++ # Check if this is an I2V pipeline (needs calibration image)+ requires_image = hasattr(c, "_requires_calibration_image") and c._requires_calibration_image()+with tqdm(range(1, total + 1), desc="cache block inputs") as pbar:for ids, prompts in c.dataloader:if isinstance(prompts, tuple):prompts = list(prompts)try:- pipe(- prompts,- guidance_scale=c.guidance_scale,- num_inference_steps=c.num_inference_steps,- generator=(- None- if c.generator_seed is None- else torch.Generator(device=pipe.device).manual_seed(c.generator_seed)- ),- )+ if requires_image:+ # I2V pipeline: pass image and prompt as kwargs+ image = c._get_calibration_image(len(prompts) if isinstance(prompts, list) else 1)+ pipe(+ image,+ prompt=prompts,+ guidance_scale=c.guidance_scale,+ num_inference_steps=c.num_inference_steps,+ generator=(+ None+ if c.generator_seed is None+ else torch.Generator(device=pipe.device).manual_seed(c.generator_seed)+ ),+ )+ else:+ pipe(+ prompts,+ guidance_scale=c.guidance_scale,+ num_inference_steps=c.num_inference_steps,+ generator=(+ None+ if c.generator_seed is None+ else torch.Generator(device=pipe.device).manual_seed(c.generator_seed)+ ),+ )except NotImplementedError:passexcept Exception as error:diff --git a/auto_round/calibration/utils.py b/auto_round/calibration/utils.pyindex 523561aef..24707564d 100644--- a/auto_round/calibration/utils.py+++ b/auto_round/calibration/utils.py@@ -45,8 +45,13 @@ def _update_inputs(inputs: dict, q_inputs: dict) -> tuple[dict, torch.Tensor]:model_context = ModelContext()if model_context.is_diffusion:- # flux transformer model's blocks will update hidden_states and encoder_hidden_statesinput_id_str = [key for key in inputs.keys() if "hidden_state" in key]+ if input_id_str == ["hidden_states"]:+ if q_inputs is not None:+ q_inputs = q_inputs.pop("hidden_states", None)+ return inputs, q_inputs++ # flux-like diffusion blocks update both hidden_states and encoder_hidden_statesif q_inputs is not None:q_inputs = {k: q_inputs.pop(k, None) for k in input_id_str}return inputs, q_inputs@@ -61,3 +66,16 @@ def _update_inputs(inputs: dict, q_inputs: dict) -> tuple[dict, torch.Tensor]:if q_inputs is not None:q_inputs = q_inputs.pop(input_id_str[0], None)return inputs, q_inputs+++def _split_inputs_diffusion(inputs: dict) -> tuple[dict, dict]:+ """Split inputs for diffusion models that only have hidden_states."""+ input_id_str = [key for key in inputs.keys() if "hidden_state" in key]+ if input_id_str == ["hidden_states"]:+ input_ids = inputs.pop("hidden_states", None)+ input_others = inputs+ return input_ids, input_others++ input_ids = {k: inputs.pop(k, None) for k in input_id_str}+ input_others = inputs+ return input_ids, input_othersdiff --git a/auto_round/compressors/data_driven.py b/auto_round/compressors/data_driven.pyindex 829408e80..c0dd82e6f 100644--- a/auto_round/compressors/data_driven.py+++ b/auto_round/compressors/data_driven.py@@ -28,6 +28,7 @@ from auto_round import envsfrom auto_round.algorithms.alg_config import AlgConfigfrom auto_round.calibration.utils import (_infer_last_cache_name,+ _split_inputs_diffusion,_update_inputs,)from auto_round.compressors.base import BaseCompressor@@ -245,6 +246,7 @@ class DataDrivenCompressor(BaseCompressor):fake_layer = _FakeDecodingLayer()fake_layer.orig_forward = fake_layer.forward+ fake_layer._true_orig_forward = lambda *a, **kw: (a, kw)fake_layer.forward = partial(self._get_block_forward_func(first_block_name), fake_layer)self.inputs = {}@@ -437,6 +439,19 @@ class DataDrivenCompressor(BaseCompressor):input_ids, input_others = self._preprocess_block_inputs(inputs)+ # For diffusion models, the heuristic split ("hidden_state" in key) may+ # place keys like encoder_hidden_states in input_ids even though they are+ # not block outputs. Move those to input_others so they persist across+ # blocks (only output keys get refreshed via reference_output each iteration).+ if self.model_context.is_diffusion and isinstance(input_ids, dict):+ first_block = get_module(model, block_names[0])+ output_config = self.quantizer.DIFFUSION_OUTPUT_CONFIGS.get(+ first_block.__class__.__name__, ["hidden_states"]+ )+ extra_keys = [k for k in list(input_ids.keys()) if k not in output_config]+ for k in extra_keys:+ input_others[k] = input_ids.pop(k)+if pbar is None:pbar = tqdm(range(0, len(block_names), nblocks))@@ -488,7 +503,7 @@ class DataDrivenCompressor(BaseCompressor):m = m.to(self.compress_context.device)card_0_in_high_risk, loss_device = False, self.compress_context.device- if len(self.compress_context.device_list) > 1:+ if len(self.compress_context.device_list) > 1 and not self.model_context.is_diffusion:from accelerate.hooks import AlignDevicesHook, add_hook_to_modulefor _n, _mod in m.named_modules():@@ -500,14 +515,20 @@ class DataDrivenCompressor(BaseCompressor):bs = self.quantizer.batch_size * self.quantizer.infer_bs_coeffif q_input is None:hook_handles = self.quantizer._register_act_max_hook(m)- reference_output = self.quantizer._get_block_outputs(m, input_ids, input_others, bs)+ reference_output = self.quantizer._get_block_outputs(+ m, input_ids, input_others, bs, device_override=loss_device+ )for h in hook_handles:h.remove()else:- reference_output = self.quantizer._get_block_outputs(m, input_ids, input_others, bs)+ reference_output = self.quantizer._get_block_outputs(+ m, input_ids, input_others, bs, device_override=loss_device+ )hook_handles = self.quantizer._register_act_max_hook(m)if hook_handles:- self.quantizer._get_block_outputs(m, q_input, input_others, bs, save_output=False)+ self.quantizer._get_block_outputs(+ m, q_input, input_others, bs, save_output=False, device_override=loss_device+ )for h in hook_handles:h.remove()@@ -541,7 +562,7 @@ class DataDrivenCompressor(BaseCompressor):q_input = None# ── Infrastructure: hook removal, device cleanup, logging ─────────- if len(self.compress_context.device_list) > 1:+ if len(self.compress_context.device_list) > 1 and not self.model_context.is_diffusion:accelerate.hooks.remove_hook_from_submodules(m)mv_module_from_gpu(m)if self.enable_torch_compile:@@ -652,7 +673,12 @@ class DataDrivenCompressor(BaseCompressor):logger.info("caching done")if self.compress_context.low_cpu_mem_usage:if self.model_context.is_model_patched and not self.compress_context.is_immediate_saving:- self._offloader(self.model_context.model, all_blocks, clear_memory=True, device_list=self.device_list)+ self._offloader(+ self.model_context.model,+ all_blocks,+ clear_memory=True,+ device_list=self.compress_context.device_list,+ )if not self._offloader.enabled:self.compress_context.low_cpu_mem_usage = Falseelse:@@ -981,7 +1007,7 @@ class CalibratedRTNCompressor(DataDrivenCompressor):self.quantizer.batch_size,self.compress_context.device,)- if len(self.compress_context.device_list) > 1:+ if len(self.compress_context.device_list) > 1 and not self.model_context.is_diffusion:from accelerate.hooks import AlignDevicesHook, add_hook_to_modulefor _, _mod in block.named_modules():diff --git a/auto_round/compressors/diffusion/README.md b/auto_round/compressors/diffusion/README.mdindex 03125e7da..96501c9b6 100644--- a/auto_round/compressors/diffusion/README.md+++ b/auto_round/compressors/diffusion/README.md@@ -60,13 +60,16 @@ auto-round \For diffusion models, currently we validate quantization on the following models, which involves quantizing the transformer component of the pipeline.-| Model | calibration dataset | Model Link |-| --- | --- | --- |-| black-forest-labs/FLUX.1-dev | COCO2014 | - |-| Tongyi-MAI/Z-Image | COCO2014 | - |-| Tongyi-MAI/Z-Image-Turb | COCO2014 | - |-| stepfun-ai/NextStep-1.1 | COCO2014 | - |-| AIDC-AI/Ovis-Image-7B | COCO2014 | - |+| Model | calibration dataset | Model Link |+|---------------|---------------------|--------------|+| black-forest-labs/FLUX.1-dev | COCO2014 | - |+| Tongyi-MAI/Z-Image | COCO2014 | - |+| Tongyi-MAI/Z-Image-Turb | COCO2014 | - |+| stepfun-ai/NextStep-1.1 | COCO2014 | - |+| AIDC-AI/Ovis-Image-7B | COCO2014 | - |+| Wan-AI/Wan2.2-I2V-A14B-Diffusers | COCO2014 | - |+| Wan-AI/Wan2.2-TI2V-5B-Diffusers | COCO2014 | - |+| Wan-AI/Wan2.2-T2V-A14B-Diffusers | COCO2014 | - |<details><summary style="font-size:17px;">Calibration Dataset</summary>diff --git a/auto_round/compressors/diffusion_mixin.py b/auto_round/compressors/diffusion_mixin.pyindex 359b3e0f3..84cde97e3 100644--- a/auto_round/compressors/diffusion_mixin.py+++ b/auto_round/compressors/diffusion_mixin.py@@ -11,12 +11,22 @@# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.# See the License for the specific language governing permissions and# limitations under the License.+import inspectimport osfrom typing import Unionimport torch+from tqdm import tqdmfrom auto_round.logger import logger+from auto_round.utils import clear_memory+from auto_round.utils.device import (+ dispatch_model_block_wise,+ dispatch_model_by_all_available_devices,+ get_major_device,+ is_auto_device_mapping,+)+from auto_round.utils.model import rename_weights_filesclass DiffusionMixin:@@ -49,12 +59,8 @@ class DiffusionMixin:self.guidance_scale = guidance_scaleself.num_inference_steps = num_inference_stepsself.generator_seed = generator_seed+ self.pipeline_call_kwargs = dict(kwargs.pop("pipeline_call_kwargs", {}) or {})- # Mirror old-arch DiffusionCompressor.__init__: when iters > 0, diffusion calibration- # cannot use batch_size > 1 for non-text modules; fold the extra batch into- # gradient_accumulate_steps so the effective sample count is unchanged.- # The authoritative batch_size lives on the AlgConfig (args[0]); kwargs may also- # carry it from AutoRoundCompatible. Patch BOTH (same pattern as MLLMMixin).iters = kwargs.get("iters", None)_alg_cfg = args[0] if args else Noneif iters is None and _alg_cfg is not None:@@ -98,12 +104,6 @@ class DiffusionMixin:# Call parent class __init__ (will be DataDrivenCompressor, ImatrixCompressor, etc)super().__init__(*args, **kwargs)- # Mirror old-arch DiffusionCompressor._align_device_and_dtype: unconditionally- # cast the full diffusion pipeline (VAE, text encoder, etc.) to the transformer's- # dtype so that calibration's pipe(...) call doesn't crash with dtype mismatches- # when the transformer is force-cast to bf16 for activation quantization.- # Note: pipe.dtype only reflects the primary component, so an equality check would- # miss mixed-dtype pipelines where e.g. the VAE is still float32.pipe = getattr(self.model_context, "pipe", None)model = getattr(self.model_context, "model", None)if pipe is not None and model is not None:@@ -121,40 +121,504 @@ class DiffusionMixin:"""return "diffusion"- def save_quantized(self, output_dir=None, format="auto_round", inplace=True, **kwargs):+ def _build_pipeline_call_kwargs(self, pipe, prompts):+ """Build kwargs for pipeline.__call__."""+ call_kwargs = {+ "guidance_scale": self.guidance_scale,+ "num_inference_steps": self.num_inference_steps,+ "generator": (+ None+ if self.generator_seed is None+ else torch.Generator(device=pipe.device).manual_seed(self.generator_seed)+ ),+ }+ call_kwargs.update(self.pipeline_call_kwargs)++ if self._requires_calibration_image():+ call_kwargs.setdefault(+ "image", self._get_calibration_image(len(prompts) if isinstance(prompts, list) else 1)+ )+ call_kwargs.setdefault("prompt", prompts)++ return call_kwargs++ def _requires_calibration_image(self) -> bool:+ """Return True when the pipeline's __call__ has a required 'image' parameter.++ I2V (image-to-video) pipelines like WanImageToVideoPipeline require a PIL/torch+ image as input. This is detected by checking whether 'image' is a positional-or-+ keyword parameter without a default value.+ """+ image_param = inspect.signature(self.model_context.pipe.__call__).parameters.get("image")+ return image_param is not None and image_param.default is inspect.Parameter.empty++ def _get_calibration_image(self, batch_size: int):+ """Return a synthetic PIL Image for I2V pipeline calibration."""+ params = inspect.signature(self.model_context.pipe.__call__).parameters+ width_param = params.get("width")+ height_param = params.get("height")+ width = (+ 832+ if width_param is None or width_param.default in (inspect.Parameter.empty, None)+ else width_param.default+ )+ height = (+ 480+ if height_param is None or height_param.default in (inspect.Parameter.empty, None)+ else height_param.default+ )+ from PIL import Image # pylint: disable=E0401++ image = Image.new("RGB", (int(width), int(height)), color=(127, 127, 127))+ if batch_size == 1:+ return image+ return [image.copy() for _ in range(batch_size)]++ def _find_additional_transformers(self):+ """Find transformer components beyond the primary one (e.g. transformer_2 in WAN)."""+ pipe = getattr(self.model_context, "pipe", None)+ if pipe is None:+ return []+ result = []+ for comp_name in pipe.components:+ comp = getattr(pipe, comp_name, None)+ if (+ comp_name.startswith("transformer")+ and comp_name != "transformer"+ and comp is not None+ and isinstance(comp, torch.nn.Module)+ ):+ result.append((comp_name, comp))+ return result++ def _align_device_and_dtype_for_secondary(self, transformer_name: str):+ """Align dtype and dispatch secondary transformer for multi-transformer pipelines."""+ pipe = getattr(self.model_context, "pipe", None)+ model = getattr(self.model_context, "model", None)+ if pipe is None or model is None:+ return++ # Cast full pipeline to transformer's dtype+ is_nextstep = hasattr(model, "config") and getattr(model.config, "model_type", None) == "nextstep"+ if not is_nextstep:+ pipe.to(model.dtype)++ # Dispatch secondary transformer to GPU(s)+ device_map = getattr(self.compress_context, "device_map", None)+ device_list = getattr(self.compress_context, "device_list", [])+ multi_device = is_auto_device_mapping(device_map) and len(device_list) > 1++ if multi_device:+ comp_device = device_list[-1]+ for comp_name in pipe.components:+ comp = getattr(pipe, comp_name, None)+ if comp is None or comp is model or not hasattr(comp, "to"):+ continue+ is_other_transformer = (+ comp_name.startswith("transformer")+ and isinstance(comp, torch.nn.Module)+ and next(comp.parameters()).device.type == "cpu"+ )+ is_other_component = not comp_name.startswith("transformer")+ if is_other_transformer or is_other_component:+ if isinstance(comp, torch.nn.Module) and hasattr(comp, "dtype") and comp.dtype != model.dtype:+ comp.to(dtype=model.dtype)+ try:+ comp.to(comp_device)+ except (NotImplementedError, RuntimeError):+ continue++ self.model_context.model = dispatch_model_block_wise(model, device_map)+ setattr(pipe, transformer_name, self.model_context.model)+ elif device_map is not None:+ target_device = get_major_device(device_map)+ pipe.to(target_device)++ @torch.no_grad()+ def calib(self, nsamples, bs):+ """Perform diffusion-specific calibration for quantization.++ Override parent's calib method to use diffusion dataset loading logic.+ The diffusion pipeline is read from ``self.model_context.pipe``.+ """+ from auto_round.compressors.diffusion.dataset import get_diffusion_dataloader++ pipe = self.model_context.pipe+ if pipe is None:+ raise ValueError(+ "Diffusion pipeline not found in model_context. " "Ensure the model was loaded as a diffusion model."+ )++ logger.warning(+ "Diffusion model will catch nsamples * num_inference_steps inputs, "+ "you can reduce nsamples or num_inference_steps if OOM or take too much time."+ )+ if isinstance(self.dataset, str):+ dataset = self.dataset.replace(" ", "")+ self.dataloader, self.batch_size, self.gradient_accumulate_steps = get_diffusion_dataloader(+ dataset=dataset,+ bs=self.batch_size,+ seed=self.seed,+ nsamples=self.nsamples,+ gradient_accumulate_steps=self.gradient_accumulate_steps,+ )+ else:+ self.dataloader = self.dataset+ total_cnt = 0++ total = nsamples if not hasattr(self.dataloader, "len") else min(nsamples, len(self.dataloader))++ # NOTE: we intentionally skip the sequential offloading check here (the guard that+ # exits when pipe is already dispatched). In new-arch diffusion, the pipe may+ # already be dispatched to multi-device in a prior calib call. The dispatch+ # state is preserved across calls, so re-dispatching or moving is unnecessary and+ # would break the existing placement.+ if (+ hasattr(self.model, "hf_device_map")+ and len(self.model.hf_device_map) > 1+ and pipe.device != self.model.device+ and torch.device(self.model.device).type in ["cuda", "xpu"]+ ):+ logger.warning(+ "Diffusion model is activated sequential model offloading. "+ "Pipe may already be dispatched from a prior calib call. "+ "Skipping re-dispatch to avoid breaking the existing placement."+ )++ if pipe.device != self.model.device:+ pipe.to(self.model.device)++ device_map = getattr(self.compress_context, "device_map", None)+ device_list = getattr(self.compress_context, "device_list", [])+ # Skip dispatch for secondary transformers+ if (+ not getattr(self, "_inputs_cached", False)+ and device_map is not None+ and is_auto_device_mapping(device_map)+ and len(device_list) > 1+ ):+ pipe_transformer = getattr(pipe, "transformer", None)+ if self.model_context.model is not pipe_transformer:+ pass # secondary transformer — skip pipeline dispatch+ else:+ pipe = dispatch_model_by_all_available_devices(pipe, device_map)++ with tqdm(range(1, total + 1), desc="cache block inputs") as pbar:+ for ids, prompts in self.dataloader:+ if isinstance(prompts, tuple):+ prompts = list(prompts)+ pipe_kwargs = self._build_pipeline_call_kwargs(pipe, prompts)+ try:+ if self._requires_calibration_image() or "prompt" in pipe_kwargs:+ # I2V pipeline: 'image' is the first positional arg, so pass+ # 'prompt' as keyword to avoid "multiple values for argument 'image'".+ pipe(**pipe_kwargs)+ else:+ pipe(prompts, **pipe_kwargs)+ except NotImplementedError:+ pass+ except Exception as error:+ raise error+ step = len(prompts)+ total_cnt += step+ pbar.update(step)+ if total_cnt >= nsamples:+ break+ if total_cnt == 0:+ logger.error(+ f"no data has been cached, please provide more data with sequence length >={self.seqlen} in the "+ f"dataset or decease the sequence length"+ )+ exit(-1)+ elif total_cnt < nsamples:+ logger.warning(+ f"Insufficient number of samples collected may affect the quantization. "+ f"target samples count is {nsamples}, while valid samples count is {total_cnt}"+ )+ if total_cnt < self.batch_size:+ raise ValueError(+ f"valid samples is less than batch_size({self.batch_size}),"+ " please adjust self.batch_size or seqlen."+ )+ max_len = (total_cnt // self.batch_size) * self.batch_size+ for k, v in self.inputs.items():+ for key in v:+ if isinstance(v[key], list) and len(v[key]) == total_cnt:+ self.inputs[k][key] = v[key][:max_len]++ # torch.cuda.empty_cache()++ def try_cache_inter_data_gpucpu(self, *args, **kwargs):+ """Skip re-caching when DiffusionMixin.quantize has already populated self.inputs.++ CalibCompressor.quantize() always calls try_cache_inter_data_gpucpu, but for+ diffusion models the inputs were already collected by the diffusion pipeline+ in DiffusionMixin.quantize(). Return the cached data directly.+ """+ if getattr(self, "_inputs_cached", False):+ self._inputs_cached = False+ return self.inputs+ if hasattr(super(), "try_cache_inter_data_gpucpu"):+ return super().try_cache_inter_data_gpucpu(*args, **kwargs)++ def quantize(self):+ """Quantize the diffusion model.++ Overrides the parent to use diffusion-specific cache_inter_data instead of+ the LLM-specific calib path. The diffusion pipeline forward is used to collect+ block inputs (via _replace_forward hooks), then those inputs are passed to the+ standard CalibCompressor quantization loop.++ For dual-transformer pipelines (e.g. WAN with transformer + transformer_2),+ this method quantizes all transformers sequentially.+ """+ from auto_round.utils import find_matching_blocks, get_block_names+ from auto_round.utils.common import flatten_list++ self.post_init()++ # Get block names and call cache_inter_data to populate self.inputs+ if bool(self.quantizer.quant_block_list):+ all_blocks = self.quantizer.quant_block_list+ else:+ all_blocks = get_block_names(self.model_context.model)+ if len(all_blocks) == 0:+ logger.warning("could not find blocks, exit with original model")+ return self.model_context.model, self.quantizer.layer_config++ if not self.has_variable_block_shape:+ to_cache_block_names = [block[0] for block in all_blocks]+ else:+ to_cache_block_names = flatten_list(all_blocks)++ # Check for additional transformers (e.g. transformer_2 in WAN)+ additional = self._find_additional_transformers()+ if not additional:+ # Single-transformer path: let calib() own pipeline dispatch.+ pipe = self.model_context.pipe+ device_map = getattr(self.compress_context, "device_map", None)+ if device_map is not None and not is_auto_device_mapping(device_map) and not isinstance(device_map, int):+ target_device = get_major_device(device_map)+ # Skip if the transformer is already on the target device to avoid+ # redundant full-model transfer that exhausts GPU memory.+ transformer = getattr(pipe, "transformer", None) or getattr(pipe, "unet", None)+ param = next(transformer.parameters(), None) if transformer else None+ skip_move = param is not None and get_major_device(str(param.device)) == target_device+ if not skip_move:+ pipe.to(target_device)++ logger.info("start to cache block inputs")+ all_inputs = self.try_cache_inter_data_gpucpu(+ to_cache_block_names,+ self.nsamples,+ layer_names=[],+ )+ self.inputs = all_inputs+ clear_memory(device_list=self.compress_context.device_list)+ self._inputs_cached = True+ return super().quantize()++ # Dual-transformer path: quantize all transformers sequentially+ logger.info("Detected multi-transformer diffusion pipeline, quantizing all transformers")++ # Ensure at least 2 inference steps so both transformers are exercised during calibration+ orig_steps = getattr(self, "num_inference_steps", None) or 50+ if orig_steps < 2:+ logger.warning(+ f"num_inference_steps={orig_steps} is too low for dual-transformer "+ f"quantization — increasing to 2 so all transformers receive calibration data."+ )+ self.num_inference_steps = 2++ # Disable low_cpu_mem_usage so quantized models stay in memory during multi-transformer+ # quantization.+ orig_low_cpu = self.compress_context.low_cpu_mem_usage+ orig_immediate_saving = self.compress_context.is_immediate_saving+ self.compress_context.low_cpu_mem_usage = False+ # Defer shard writing until all transformers are quantized. Immediate saving+ # offloads the primary transformer to meta during the first pass, which breaks+ # later calibration when the pipeline switches to transformer_2.+ self.compress_context.is_immediate_saving = False++ # Store primary transformer state+ primary_model = self.model_context.model+ primary_layer_config = dict(self.layer_config) if self.layer_config else {}+ primary_quant_block_list = list(self.quant_block_list) if self.quant_block_list else []+ quantized_extras = {}++ # Quantize primary transformer+ logger.info("start to cache block inputs for primary transformer")+ all_inputs = self.try_cache_inter_data_gpucpu(+ to_cache_block_names,+ self.nsamples,+ layer_names=[],+ )+ self.inputs = all_inputs+ clear_memory(device_list=self.compress_context.device_list)+ self._inputs_cached = True+ super().quantize()++ # Clear stale hf_device_map so cache_inter_data can re-dispatch for next transformer+ if hasattr(primary_model, "hf_device_map"):+ delattr(primary_model, "hf_device_map")++ # Quantize additional transformers+ for comp_name, transformer in additional:+ logger.info(f"Quantizing {comp_name}")++ # Reset quantization state for new transformer+ self.model_context.model = transformer+ self.model_context.quantized = False+ self._post_init_done = False++ # Re-align device/dtype and dispatch pipeline for secondary transformer+ self._align_device_and_dtype_for_secondary(comp_name)++ # Re-run post_init to set up quantizer for new model+ self.post_init()++ # Get block names for new transformer+ all_blocks = get_block_names(self.model_context.model)+ self.quant_block_list = find_matching_blocks(self.model_context.model, all_blocks, None)+ self.layer_config = {}++ # Get new block names for caching+ if bool(self.quantizer.quant_block_list):+ all_blocks = self.quantizer.quant_block_list+ else:+ all_blocks = get_block_names(self.model_context.model)+ if len(all_blocks) == 0:+ logger.warning(f"could not find blocks in {comp_name}, skipping")+ continue++ if not self.has_variable_block_shape:+ to_cache_block_names = [block[0] for block in all_blocks]+ else:+ to_cache_block_names = flatten_list(all_blocks)++ logger.info(f"start to cache block inputs for {comp_name}")+ all_inputs = self.try_cache_inter_data_gpucpu(+ to_cache_block_names,+ self.nsamples,+ layer_names=[],+ )+ self.inputs = all_inputs+ clear_memory(device_list=self.compress_context.device_list)+ self._inputs_cached = True+ super().quantize()++ # Store quantized transformer+ quantized_extras[comp_name] = (self.model_context.model, dict(self.layer_config))+ # Also update the pipeline to reference the quantized transformer+ setattr(self.model_context.pipe, comp_name, self.model_context.model)++ # Restore primary transformer state+ self.model_context.model = primary_model+ self.model_context.quantized = True+ self.layer_config = primary_layer_config+ self.quant_block_list = primary_quant_block_list+ self._quantized_transformers = quantized_extras+ self.compress_context.low_cpu_mem_usage = orig_low_cpu+ self.compress_context.is_immediate_saving = orig_immediate_saving+ self.num_inference_steps = orig_steps++ return self.model_context.model, self.quantizer.layer_config++ def save_quantized(self, output_dir=None, format="auto_round", inplace=True, return_folders=False, **kwargs):"""Save the quantized model to the specified output directory in the specified format.+ For multi-transformer pipelines, all quantized transformers are saved.+Args:output_dir (str, optional): The directory to save the quantized model. Defaults to None.format (str, optional): The format in which to save the model. Defaults to "auto_round".inplace (bool, optional): Whether to modify the model in place. Defaults to True.+ return_folders (bool, optional): Whether to return the save folder paths. Defaults to False.**kwargs: Additional keyword arguments specific to the export format.Returns:- object: The compressed model object.+ object: The compressed model object, or (compressed_model, folders) if return_folders is True."""if output_dir is None:- return super().save_quantized(output_dir, format=format, inplace=inplace, **kwargs)+ return super().save_quantized(+ output_dir, format=format, inplace=inplace, return_folders=return_folders, **kwargs+ )pipe = self.model_context.pipe+ quantized_transformers = getattr(self, "_quantized_transformers", {})compressed_model = None+ folders = []+ has_multiple_quantized_transformers = bool(quantized_transformers)++ # Handle multi-format (convert string to list if needed)+ _format = format+ if isinstance(_format, str):+ from auto_round.formats import get_formats++ _format = get_formats(_format, self)+for name in pipe.components.keys():val = getattr(pipe, name)sub_module_path = (os.path.join(output_dir, name) if os.path.basename(os.path.normpath(output_dir)) != name else output_dir)- if (- hasattr(val, "config")- and hasattr(val.config, "_name_or_path")- and val.config._name_or_path == self.model.config._name_or_path- ):+ target_output_dir = (+ sub_module_path+ if has_multiple_quantized_transformers or not self.compress_context.is_immediate_saving+ else output_dir+ )+ if name in quantized_transformers:+ # Save secondary quantized transformer+ saved_model = self.model_context.model+ saved_lc = self.layer_config+ saved_immediate_saving = self.compress_context.is_immediate_saving+ self.model_context.model, self.layer_config = quantized_transformers[name]+ saved_subfolder = getattr(self.model_context.model, "_autoround_pipeline_subfolder", None)+ if has_multiple_quantized_transformers:+ self.compress_context.is_immediate_saving = False+ if saved_subfolder is not None:+ delattr(self.model_context.model, "_autoround_pipeline_subfolder")compressed_model = super().save_quantized(- output_dir=sub_module_path if not self.compress_context.is_immediate_saving else output_dir,- format=format,+ output_dir=target_output_dir,+ format=_format,inplace=inplace,+ return_folders=False,**kwargs,)+ self.compress_context.is_immediate_saving = saved_immediate_saving+ if saved_subfolder is not None:+ self.model_context.model._autoround_pipeline_subfolder = saved_subfolder+ self.model_context.model = saved_model+ self.layer_config = saved_lc+ elif val is self.model_context.model:+ # Save primary quantized transformer+ saved_immediate_saving = self.compress_context.is_immediate_saving+ saved_subfolder = getattr(self.model_context.model, "_autoround_pipeline_subfolder", None)+ if has_multiple_quantized_transformers:+ self.compress_context.is_immediate_saving = False+ if saved_subfolder is not None:+ delattr(self.model_context.model, "_autoround_pipeline_subfolder")+ compressed_model = super().save_quantized(+ output_dir=target_output_dir,+ format=_format,+ inplace=inplace,+ return_folders=False,+ **kwargs,+ )+ self.compress_context.is_immediate_saving = saved_immediate_saving+ if saved_subfolder is not None:+ self.model_context.model._autoround_pipeline_subfolder = saved_subfolderelif val is not None and hasattr(val, "save_pretrained"):val.save_pretrained(sub_module_path)- pipe.config.save_pretrained(output_dir)+ continue++ if name.startswith("transformer"):+ rename_weights_files(target_output_dir)++ folders.append(target_output_dir)++ pipe.save_config(output_dir)++ if return_folders:+ return compressed_model, foldersreturn compressed_modeldiff --git a/auto_round/context/model.py b/auto_round/context/model.pyindex a29db4073..2cd5273d1 100644--- a/auto_round/context/model.py+++ b/auto_round/context/model.py@@ -76,6 +76,9 @@ class ModelContext(BaseContext):self.is_diffusion = Falseself.is_model_patched = Falseself.is_moe_model = False+ # Set by CalibCompressor._replace_forward; used by recover_forward to detect+ # new-arch diffusion mode where positional wrapper must be stripped after caching.+ self._has_true_orig_forward_set = Falseassert model is not None, "model must be provided for ModelContext"self.model = model@@ -282,14 +285,44 @@ class ModelContext(BaseContext):self.hook_handles = hook_handles- def recover_forward(self):- """Recovers the forward function."""+ def recover_forward(self, restore_positional_wrapper=None):+ """Recovers the forward function.++ Args:+ restore_positional_wrapper: If True, restores forward to the wrapped version+ (needed for LLM calibration where positional wrapper is used during quantization).+ If False, restores to the true original forward (needed for diffusion).+ If None (default), auto-detects: uses False for diffusion models.+ """assert self._init_model, "should load and initialize model first"+ # Auto-detect for diffusion: when _true_orig_forward is present (set by+ # CalibCompressor._replace_forward), we are in new-arch diffusion mode where+ # the positional wrapper must be fully removed after caching.+ if restore_positional_wrapper is None:+ restore_positional_wrapper = not getattr(self, "_has_true_orig_forward_set", False)+ if not restore_positional_wrapper:+ logger.debug("recover_forward: auto-detected diffusion mode, stripping positional wrapper")+for n, m in self.model.named_modules():if hasattr(m, "orig_forward"):- m.forward = m.orig_forward- delattr(m, "orig_forward")+ true_orig = getattr(m, "_true_orig_forward", m.orig_forward)+ if restore_positional_wrapper:+ # Restore orig_forward so that any wrapper (e.g. from+ # wrap_block_forward_positional_to_kwargs) can still access it.+ # The wrapper holds a closure reference to orig_forward.+ m.forward = getattr(m, "_wrapped_forward_before_replace", m.orig_forward)+ m.orig_forward = true_orig+ else:+ # Full recovery: restore the true original forward. Used for diffusion+ # where the positional wrapper must be fully removed after caching.+ m.forward = true_orig+ # Keep _true_orig_forward so the wrapped forward's base_hook can+ # still call it during quantization tuning.+ m._true_orig_forward = true_orig+ delattr(m, "orig_forward")+ if hasattr(m, "_wrapped_forward_before_replace"):+ delattr(m, "_wrapped_forward_before_replace")for hook_handle in self.hook_handles:hook_handle.remove()self.hook_handles = []diff --git a/auto_round/eval/evaluation.py b/auto_round/eval/evaluation.pyindex 5933bbc0d..2288dcafc 100644--- a/auto_round/eval/evaluation.py+++ b/auto_round/eval/evaluation.py@@ -415,6 +415,10 @@ def run_model_evaluation(model, tokenizer, autoround, folders, formats, device_s# Handle diffusion models separatelyif getattr(autoround, "diffusion", False):+ prompt = getattr(args, "prompt", None)+ prompt_file = getattr(args, "prompt_file", None)+ if prompt is None and prompt_file is None:+ returnevaluate_diffusion_model(args, autoround=autoround, model=model)returndiff --git a/auto_round/export/utils.py b/auto_round/export/utils.pyindex 837a9f073..85765d11b 100644--- a/auto_round/export/utils.py+++ b/auto_round/export/utils.py@@ -75,6 +75,18 @@ def _resolve_pipeline_source_dir(model: nn.Module) -> str | None:return None+def _resolve_model_source_dir(model: nn.Module) -> str | None:+ candidates = [+ getattr(model, "name_or_path", None),+ getattr(getattr(model, "config", None), "_name_or_path", None),+ getattr(getattr(model, "config", None), "name_or_path", None),+ ]+ for candidate in candidates:+ if isinstance(candidate, str) and candidate:+ return candidate+ return None++def _copy_pipeline_artifact(model_dir: str, relative_path: str, output_dir: str) -> None:target_path = os.path.join(output_dir, relative_path)os.makedirs(os.path.dirname(target_path), exist_ok=True)@@ -212,16 +224,14 @@ def save_model(else:model.save_pretrained(save_dir, max_shard_size=max_shard_size, safe_serialization=safe_serialization)+ source_dir = _resolve_model_source_dir(model)+# Allow disabling copy_missing_tensors_from_source via env var AR_DISABLE_COPY_MTP_WEIGHTS, default enabledif not envs.AR_DISABLE_COPY_MTP_WEIGHTS:try:- if (- hasattr(model, "config")- and hasattr(model.config, "_name_or_path")- and model.config.name_or_path is not None # set None for tiny model- ):+ if source_dir is not None:copy_missing_tensors_from_source(- source_dir=model.config._name_or_path,+ source_dir=source_dir,target_dir=save_dir,)except Exception as e:@@ -245,11 +255,7 @@ def save_model(json.dump(model.config.quantization_config, f, indent=2)try:- if (- hasattr(model, "config")- and hasattr(model.config, "_name_or_path")- and model.config.name_or_path is not None # set None for tiny model- ):+ if source_dir is not None:copy_python_files_from_model_cache(model, save_dir)except Exception as e:logger.warning("Skipping source model Python file copy due to error: %s", e)diff --git a/auto_round/modeling/fused_moe/moe_experts_interface.py b/auto_round/modeling/fused_moe/moe_experts_interface.pyindex d8fc9ab46..854d9e186 100644--- a/auto_round/modeling/fused_moe/moe_experts_interface.py+++ b/auto_round/modeling/fused_moe/moe_experts_interface.py@@ -321,11 +321,16 @@ def _detect_expert_projections(module: nn.Module) -> dict[str, dict]:if param is not None and isinstance(param, nn.Parameter) and param.dim() == 3:detected[proj_name] = config- # If no known patterns found, scan for any 3D Parameter (future-proofing)+ # If no known patterns found, scan for any 3D Parameter (future-proofing),+ # but exclude known non-expert 3D parameters that exist on some transformer+ # architectures (e.g. scale_shift_table on WanTransformer3DModel).+ _NON_EXPERT_3D_PARAMS = frozenset(["scale_shift_table"])if not detected:for attr_name in dir(module):if attr_name.startswith("_"):continue+ if attr_name in _NON_EXPERT_3D_PARAMS:+ continueparam = getattr(module, attr_name, None)if param is not None and isinstance(param, nn.Parameter) and param.dim() == 3:# Use default config for unknown projectionsdiff --git a/auto_round/special_model_handler.py b/auto_round/special_model_handler.pyindex 6f11b454c..68ac5b540 100644--- a/auto_round/special_model_handler.py+++ b/auto_round/special_model_handler.py@@ -58,6 +58,7 @@ SPECIAL_SHARED_CACHE_KEYS = {"Gemma3ForConditionalGeneration": ("position_embeddings_global", "position_embeddings_local")}SPECIAL_SHARED_CACHE_KEYS["MiniMaxText01ForCausalLM"] = ("slope_rate",)+SPECIAL_SHARED_CACHE_KEYS["WanTransformer3DModel"] = ("rotary_emb",)MISTRAL_3_2_MODELS = ["Mistral-Small-3.2", "Magistral-Small", "Devstral-Small"]diff --git a/auto_round/utils/device.py b/auto_round/utils/device.pyindex e90d78431..f864a941e 100644--- a/auto_round/utils/device.py+++ b/auto_round/utils/device.py@@ -1489,7 +1489,8 @@ def dispatch_model_block_wise(model: torch.nn.Module, device_map: str, max_mem_rmax_memory=new_max_memory,no_split_module_classes=no_split_modules,)- model.tie_weights()+ if hasattr(model, "tie_weights"):+ model.tie_weights()device_map = infer_auto_device_map(model, max_memory=new_max_memory, no_split_module_classes=no_split_modules)if len(devices) > 1 and "cpu" in device_map.values():logger.warning(@@ -1918,20 +1919,83 @@ def dispatch_model_by_all_available_devices(# move the entire pipeline to the (first) device.pipe.to(devices[0] if devices else "cuda:0")return pipe- # Multi-device path: recursively dispatch the main sub-model,- # then move all remaining pipeline components to the primary device.+ # Multi-device path: dispatch the main sub-model across devices,+ # reserving memory on the primary device for non-target components+ # (text encoder, VAE, etc.) to avoid OOM.main_model = getattr(pipe, main_attr)- dispatched = dispatch_model_by_all_available_devices(main_model, _device_map)- setattr(pipe, main_attr, dispatched)primary_device = devices[0]+ # Place non-main components on the last device+ comp_device = devices[-1]for attr, component in pipe.components.items():if attr == main_attr:continue- if isinstance(component, torch.nn.Module):+ if not isinstance(component, torch.nn.Module):+ continue+ # Align dtype on CPU first to avoid wasting GPU memory+ if hasattr(component, "dtype") and component.dtype != main_model.dtype:try:- component.to(primary_device)+ component.to(dtype=main_model.dtype)except Exception:pass+ try:+ component.to(comp_device)+ except (NotImplementedError, RuntimeError):+ pass++ from auto_round.utils.common import normalize_no_split_modules++ no_split_modules = normalize_no_split_modules(getattr(main_model, "_no_split_modules", []))++ # dispatch_model_block_wise queries free memory after non-main+ # components are already placed, so the budget naturally accounts for+ # them. Use the same approach here.+ dispatched = dispatch_model_block_wise(main_model, device_map)+ setattr(pipe, main_attr, dispatched)++ # Install manual pre/post hooks to move tensors.+ unique_devices = set()+ if hasattr(dispatched, "hf_device_map"):+ unique_devices = {v for v in dispatched.hf_device_map.values() if v not in ("cpu", "disk")}+ if len(unique_devices) <= 1:+ execution_device = primary_device+ try:+ execution_device = next(dispatched.parameters()).device+ except StopIteration:+ execution_device = torch.device(primary_device)++ if hasattr(dispatched, "_hf_hook") and hasattr(dispatched._hf_hook, "execution_device"):+ dispatched._hf_hook.execution_device = execution_device++ if not getattr(dispatched, "_autoround_align_inputs_hook_installed", False):+ _first_param_device = execution_device+ _pipeline_device = torch.device(comp_device)++ def _align_all_inputs_pre_hook(module, args, kwargs):+ try:+ target = next(module.parameters()).device+ except StopIteration:+ target = _first_param_device+ new_args = tuple(a.to(target) if isinstance(a, torch.Tensor) else a for a in args)+ new_kwargs = {k: v.to(target) if isinstance(v, torch.Tensor) else v for k, v in kwargs.items()}+ return new_args, new_kwargs++ def _move_outputs_back_hook(module, input, output):+ def _to_device(obj, device):+ if isinstance(obj, torch.Tensor):+ return obj.to(device) if obj.device != device else obj+ if isinstance(obj, (tuple, list)):+ converted = [_to_device(o, device) for o in obj]+ return type(obj)(converted)+ if isinstance(obj, dict):+ return {k: _to_device(v, device) for k, v in obj.items()}+ return obj++ return _to_device(output, _pipeline_device)++ dispatched.register_forward_pre_hook(_align_all_inputs_pre_hook, with_kwargs=True)+ dispatched.register_forward_hook(_move_outputs_back_hook)+ dispatched._autoround_align_inputs_hook_installed = True+return pipeif device_map is None:diff --git a/auto_round/utils/model.py b/auto_round/utils/model.pyindex db7f2294f..8569fab9c 100644--- a/auto_round/utils/model.py+++ b/auto_round/utils/model.py@@ -732,7 +732,7 @@ def diffusion_load_model(component_config = json.load(file)torch_dtype[k] = component_config.get("torch_dtype", "auto")- pipe = pipelines.auto_pipeline.AutoPipelineForText2Image.from_pretrained(+ pipe = pipelines.pipeline_utils.DiffusionPipeline.from_pretrained(pretrained_model_name_or_path, torch_dtype=torch_dtype)pipe_config = pipe.load_config(pretrained_model_name_or_path)@@ -767,6 +767,20 @@ def diffusion_load_model(# non-meta model uses model.save_pretrained for model and config savingsetattr(model, "save_pretrained", partial(model_save_pretrained, model))++ for comp_name in pipe.components:+ comp = getattr(pipe, comp_name, None)+ if (+ comp_name.startswith("transformer")+ and comp_name != "transformer"+ and comp is not None+ and isinstance(comp, torch.nn.Module)+ ):+ setattr(+ comp.config, "save_pretrained", partial(config_save_pretrained, comp.config, "config.json", model=comp)+ )+ setattr(comp, "save_pretrained", partial(model_save_pretrained, comp))+return pipe, model.to(device)@@ -2000,14 +2014,18 @@ def wrap_block_forward_positional_to_kwargs(base_hook):positional args to keyword args, all inputs are properly accumulatedacross calibration samples."""- _param_names = None+ _param_names_cache: dict = {}def forward(m, hidden_states=None, *positional_inputs, **kwargs):- nonlocal _param_namesif positional_inputs:- if _param_names is None:- sig = inspect.signature(m.orig_forward)- _param_names = [p for p in sig.parameters.keys() if p != "self"]+ m_id = id(m)+ if m_id not in _param_names_cache:+ # Prefer _true_orig_forward (set by new-arch CalibCompressor._replace_forward)+ # over orig_forward (which points to the wrapped forward after wrapping).+ sig_target = getattr(m, "_true_orig_forward", None) or m.orig_forward+ sig = inspect.signature(sig_target)+ _param_names_cache[m_id] = [p for p in sig.parameters.keys() if p != "self"]+ _param_names = _param_names_cache[m_id]for i, val in enumerate(positional_inputs):param_idx = i + 1 # hidden_states is params[0]if param_idx < len(_param_names):