PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Fix few merge errors (#1817)
Heng Guo706ac7e39861
diff --git a/auto_round/algorithms/quantization/base.py b/auto_round/algorithms/quantization/base.pyindex b0e0cbf6b..765991844 100644--- a/auto_round/algorithms/quantization/base.py+++ b/auto_round/algorithms/quantization/base.py@@ -19,12 +19,12 @@ from typing import Unionimport torchfrom auto_round.algorithms.quantization.config import QuantizationConfig+from auto_round.algorithms.quantization.utils import register_act_max_hooksfrom auto_round.compressors.utils import (block_forward,- check_need_act_calibration,+ immediate_pack,)from auto_round.data_type import QUANT_FUNC_WITH_DTYPE-from auto_round.data_type.utils import reshape_pad_tensor_by_group_sizefrom auto_round.logger import loggerfrom auto_round.utils import (INNER_SUPPORTED_LAYER_TYPES,@@ -32,7 +32,11 @@ from auto_round.utils import (check_to_quantized,clear_memory,compile_func,+ convert_module_to_hp_if_necessary,+ get_module,+ set_module,)+from auto_round.wrapper import WrapperLinearclass BaseQuantizers:@@ -172,66 +176,8 @@ class BaseQuantizers:return getattr(self.model_context, "amp_dtype", torch.float32)- def _register_act_max_hook(self, model):-- def get_act_max_hook(module, input, output):- if isinstance(input, (tuple, list)):- input = input[0]- if input.numel() == 0:- return # as no needs for act_max update- input, _, _ = reshape_pad_tensor_by_group_size(input, self.act_group_size)- act_max = torch.max(torch.abs(input), dim=-1).values- if not hasattr(module, "act_max") or module.act_max.numel() == 0:- module.act_max = act_max- if self.config.is_act_nv_fp: ## for nvfp per-tensor input_global_scale calculation usage- max_val = act_max.max()- module.act_max = max_val.unsqueeze(0) if max_val.dim() == 0 else max_val- else:- act_max = act_max.to(module.act_max.device)- if self.config.is_act_nv_fp: ## for nvfp per-tensor input_global_scale calculation usage- max_val = torch.max(act_max.max(), module.act_max.max())- module.act_max = max_val.unsqueeze(0) if max_val.dim() == 0 else max_val- else:- module.act_max = torch.max(act_max, module.act_max)-- hook_handles = []- # for single layers out of blocks, like lm_head- if isinstance(model, SUPPORTED_LAYER_TYPES):- m = model- if (- hasattr(m, "act_dynamic")- and check_need_act_calibration(m.act_dynamic, m.act_data_type, m.act_bits)- and check_to_quantized(m)- ):- hook = m.register_forward_hook(get_act_max_hook)- hook_handles.append(hook)- return hook_handles-- for n, m in model.named_modules():- if (- hasattr(m, "act_dynamic")- and check_need_act_calibration(m.act_dynamic, m.act_data_type, m.act_bits)- and check_to_quantized(m)- ):- hook = m.register_forward_hook(get_act_max_hook)- hook_handles.append(hook)- continue-- # for whole model, RTN- if n in self.layer_config:- config = self.layer_config[n]- act_dynamic = config.get("act_dynamic", True)- act_data_type = config.get("act_data_type", None)- act_bits = config.get("act_bits", 16)- if (- config["bits"] <= 8- and check_need_act_calibration(act_dynamic, act_data_type, act_bits)- and check_to_quantized(config)- ):- hook = m.register_forward_hook(get_act_max_hook)- hook_handles.append(hook)- continue- return hook_handles+ def register_calibration_hooks(self, model, *, act_max: bool = True, imatrix: bool = True):+ return register_act_max_hooks(self, model) if act_max else []@torch.inference_mode()def _quantize_embedding_layer(self):@@ -343,6 +289,94 @@ class BaseQuantizers:"""raise NotImplementedError("quantize_block must be implemented in subclasses of BaseQuantizers")+ @torch.no_grad()+ def quantize_layer_via_rtn(self, layer_name: str, disable_opt_rtn: bool | None = None) -> None:+ """Quantize one layer with RTN and handle optional immediate pack/save."""+ layer = get_module(self.model, layer_name)+ layer = convert_module_to_hp_if_necessary(layer, self.model_context.amp_dtype, self.compress_context.device)+ set_module(self.model, layer_name, layer)+ tuning_device = layer.tuning_device if hasattr(layer, "tuning_device") else self.compress_context.device+ if (+ self.compress_context.is_immediate_packing+ and self.compress_context.formats[0].is_gguf()+ and not getattr(self.config, "disable_opt_rtn", False)+ ):+ layer = layer.to(tuning_device)+ layer.scale = None+ layer.zp = None+ else:+ try:+ if disable_opt_rtn is None:+ disable_opt_rtn = bool(getattr(self.config, "disable_opt_rtn", False))+ if (+ not disable_opt_rtn+ and getattr(self.config, "orig_disable_opt_rtn", None) is None+ and self.model_context.is_moe_model+ and "expert" in layer.global_name+ and "shared_expert" not in layer.global_name+ and self.config.super_bits is None+ ):+ disable_opt_rtn = True+ logger.warning_once(+ "MoE layer detected: optimized RTN is disabled for efficiency. "+ "Use `--enable_opt_rtn` to force-enable it for MoE layers."+ )+ layer = layer.to(tuning_device)+ layer = WrapperLinear(+ layer,+ device=tuning_device,+ enable_minmax_tuning=False,+ enable_norm_bias_tuning=False,+ enable_round_tuning=False,+ enable_torch_compile=self.compress_context.enable_torch_compile,+ disable_opt_rtn=disable_opt_rtn,+ iters=0,+ )+ layer = layer.unwrapper({})+ except torch.OutOfMemoryError:+ cuda_error_msg = traceback.format_exc()+ layer = layer.orig_layer if hasattr(layer, "orig_layer") else layer+ try:+ logger.error(cuda_error_msg)+ logger.warning("falling back to CPU.")+ layer.to("cpu")+ layer = WrapperLinear(+ layer,+ enable_minmax_tuning=False,+ enable_norm_bias_tuning=False,+ enable_round_tuning=False,+ enable_torch_compile=self.compress_context.enable_torch_compile,+ iters=0,+ )+ layer = layer.unwrapper({})+ except Exception:+ raise++ set_module(self.model, layer_name, layer)+ self._immediate_pack_and_save_module(layer_name)++ def _immediate_pack_and_save_module(self, module_name):+ from auto_round.compressors.shard_writer import ShardWriter++ shard_writer = ShardWriter.get_shard_writer()+ to_cpu = self.compress_context.low_gpu_mem_usage+ module = get_module(self.model, module_name)+ if self.compress_context.is_immediate_packing:+ immediate_pack(module_name, self.layer_config)+ if to_cpu:+ module = module.to("cpu")+ packed_module = get_module(self.model, module_name)+ set_module(self.model, module_name, packed_module.to("cpu"))+ else:+ if to_cpu:+ module = module.to("cpu")+ set_module(self.model, module_name, module)+ if self.compress_context.is_immediate_saving:+ module = get_module(self.model, module_name)+ module.to("cpu")+ shard_writer.write(module, module_name, False)+ module.to("meta")+def quantize_layer(self, layer_name: str, **kwargs):"""Quantizes a single layer of the model.@@ -352,14 +386,19 @@ class BaseQuantizers:"""raise NotImplementedError("quantize_layer must be implemented in subclasses of BaseQuantizers")- def quantize_layer_outside_block(self, layer_name: str, **kwargs):+ def quantize_layer_outside_block(self, layer_name: str, input_ids=None, **kwargs):"""Quantizes a single layer of the model outside of a block.Args:layer_name (str): The name of the layer to quantize. The layer module isretrieved internally via get_module(model, layer_name).+ input_ids: Optional calibration inputs for data-driven outside-layer quantization."""- raise NotImplementedError("quantize_layer_outside_block must be implemented in subclasses of BaseQuantizers")+ dtype = kwargs.pop("dtype", None)+ if dtype is not None:+ layer = get_module(self.model, layer_name)+ set_module(self.model, layer_name, layer.to(dtype))+ self.quantize_layer_via_rtn(layer_name, **kwargs)@torch.no_grad()def _get_block_outputs(@@ -543,9 +582,18 @@ class BaseQuantizers:# Shared keys are stored once (not per-sample), often wrapped in a# 1-element list by the caching hook. Unwrap so the model receives# the raw value (e.g. (cos, sin) tuple, not [(cos, sin)]).+ # Exception: if the hook detected that the "shared" key actually varies+ # per sample (e.g. position_embeddings in a VLM visual encoder), it+ # upgrades the storage to a per-sample list with >1 elements.val = input_others[key]if isinstance(val, list) and len(val) == 1:current_input_others[key] = val[0]+ elif isinstance(val, list) and len(val) > 1:+ # Per-sample storage for a nominally-shared key that varies across+ # calibration samples (e.g. position_embeddings in Qwen2-VL visual+ # encoder blocks where each image has a different patch count).+ idx = int(indices[0]) if len(indices) == 1 else 0+ current_input_others[key] = val[idx] if idx < len(val) else val[0]else:current_input_others[key] = valelif not isinstance(input_others[key], (str, bool, type(None))):diff --git a/auto_round/algorithms/quantization/rtn/quantizer.py b/auto_round/algorithms/quantization/rtn/quantizer.pyindex af8312f26..d1c2c0779 100644--- a/auto_round/algorithms/quantization/rtn/quantizer.py+++ b/auto_round/algorithms/quantization/rtn/quantizer.py@@ -11,7 +11,6 @@# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.# See the License for the specific language governing permissions and# limitations under the License.-import tracebackfrom collections import defaultdictfrom typing import Any, Callable, Optional, Union@@ -21,7 +20,7 @@ import torchfrom auto_round.algorithms.quantization.base import BaseQuantizersfrom auto_round.algorithms.quantization.rtn.config import RTNConfigfrom auto_round.algorithms.quantization.sign_round.quantizer import SignRoundQuantizer-from auto_round.compressors.shard_writer import ShardWriter+from auto_round.algorithms.quantization.utils import register_imatrix_hooksfrom auto_round.compressors.utils import (IndexSampler,block_forward,@@ -29,7 +28,6 @@ from auto_round.compressors.utils import (check_skippable_keywords,collect_best_params,get_shared_keys,- immediate_pack,infer_bits_by_data_type,init_cache,reset_params,@@ -39,7 +37,6 @@ from auto_round.data_type.utils import update_block_global_scale_if_neededfrom auto_round.logger import loggerfrom auto_round.utils import (check_to_quantized,- convert_module_to_hp_if_necessary,get_lm_head_name,get_module,htcore,@@ -56,7 +53,7 @@ from auto_round.utils.device import (set_auto_device_map_for_block_with_tuning,set_non_auto_device_map,)-from auto_round.wrapper import WrapperLinear, WrapperMultiblock, unwrapper_block, unwrapper_layer, wrapper_block+from auto_round.wrapper import WrapperMultiblock, unwrapper_block, unwrapper_layer, wrapper_blockclass RTNQuantizer(BaseQuantizers):@@ -100,110 +97,10 @@ class RTNQuantizer(BaseQuantizers):@torch.no_grad()def quantize_layer(self, name: str, dtype: torch.dtype = None) -> None:- """Quantizes a layer using RTN (Round-To-Nearest) if available.-- This function attempts to quantize a layer by switching its data type to a- `rtn_*` version if supported, then wraps and unwraps the module to apply- quantization. If GPU memory is insufficient, it falls back to CPU.-- If packing is enabled (`immediate_packing`), the function will also export- the quantized layer to the appropriate backend format.-- Args:- name (str): Name of the layer to quantize.-- Raises:- RuntimeError: If quantization fails for reasons unrelated to memory.- """-- m = get_module(self.model, name)if dtype is not None:- m = m.to(dtype)-- m = convert_module_to_hp_if_necessary(m, self.model_context.amp_dtype, self.compress_context.device)- set_module(self.model, name, m)- tuning_device = m.tuning_device if hasattr(m, "tuning_device") else self.compress_context.device- # Step 1: let gguf merge layers or rename module first and we will handle the RTN is gguf specific logic- if (- self.compress_context.is_immediate_packing- and self.compress_context.formats[0].is_gguf()- and not getattr(self.config, "disable_opt_rtn", False)- ):- m = m.to(tuning_device)- m.scale = None- m.zp = None- else:- try:- disable_opt_rtn = bool(getattr(self.config, "disable_opt_rtn", False))- if (- not disable_opt_rtn- and self.config.orig_disable_opt_rtn is None- and self.model_context.is_moe_model- and "expert" in m.global_name- and "shared_expert" not in m.global_name- and self.config.super_bits is None # GGUF still uses the optimized RTN for MoE layers- ):- disable_opt_rtn = True- logger.warning_once(- "MoE layer detected: optimized RTN is disabled for efficiency. "- "Use `--enable_opt_rtn` to force-enable it for MoE layers."- )- m = m.to(tuning_device)- m = WrapperLinear(- m,- device=tuning_device,- enable_minmax_tuning=False,- enable_norm_bias_tuning=False,- enable_round_tuning=False,- enable_torch_compile=self.compress_context.enable_torch_compile,- disable_opt_rtn=disable_opt_rtn,- iters=0,- )- m = m.unwrapper({})- except torch.OutOfMemoryError:- cuda_error_msg = traceback.format_exc()- m = m.orig_layer if hasattr(m, "orig_layer") else m- try:- logger.error(cuda_error_msg)- logger.warning("falling back to CPU.")- m.to("cpu")- m = WrapperLinear(- m,- enable_minmax_tuning=False,- enable_norm_bias_tuning=False,- enable_round_tuning=False,- enable_torch_compile=self.compress_context.enable_torch_compile,- iters=0,- )- m = m.unwrapper({})- except Exception as e:- raise-- set_module(self.model, name, m)- self._immediate_pack_and_save_module(name)-- def _immediate_pack_and_save_module(self, module_name):- shard_writer = ShardWriter.get_shard_writer()- to_cpu = self.compress_context.low_gpu_mem_usage- module = get_module(self.model, module_name)- if self.compress_context.is_immediate_packing: # For gguf, packing conducts on block level- immediate_pack(module_name, self.layer_config)- if to_cpu:- module = module.to("cpu")- packed_module = get_module(self.model, module_name)- set_module(self.model, module_name, packed_module.to("cpu"))- else:- if to_cpu:- module = module.to("cpu")- set_module(self.model, module_name, module)- if self.compress_context.is_immediate_saving:- module = get_module(self.model, module_name)- module.to("cpu")- shard_writer.write(module, module_name, False)- # Free RAM immediately: the data is now in the shard-writer buffer- # (and will be flushed to disk). Keeping it also in the model tree- # causes linear RAM growth for large models.- module.to("meta")+ layer = get_module(self.model, name)+ set_module(self.model, name, layer.to(dtype))+ self.quantize_layer_via_rtn(name)class OptimizedRTNQuantizer(RTNQuantizer):@@ -213,11 +110,15 @@ class OptimizedRTNQuantizer(RTNQuantizer):self.data_type = config.data_typeself.group_size = config.group_sizeself.infer_bs_coeff = config.infer_bs_coeff+ self.enable_imatrix = getattr(config, "enable_imatrix", False)self.enable_alg_ext = True- def quantize_layer_outside_block(self, *args, **kwargs):- return self.quantize_layer(*args, **kwargs)+ def register_calibration_hooks(self, model, *, act_max: bool = True, imatrix: bool = True):+ hook_handles = super().register_calibration_hooks(model, act_max=act_max, imatrix=imatrix)+ if imatrix and self.enable_imatrix:+ hook_handles.extend(register_imatrix_hooks(self, model, with_count=True))+ return hook_handles@torch.no_grad()def quantize_block(@@ -225,9 +126,9 @@ class OptimizedRTNQuantizer(RTNQuantizer):):"""Apply imatrix-informed RTN quantization to a block.- Pure-algorithm entry point. All infrastructure (device placement,- act-max hook registration, imatrix collection, cleanup) is handled- by the Compressor before calling this method.+ Pure-algorithm entry point. Device placement and cleanup are handled+ by the Compressor; act-max and imatrix hook registration are owned by+ the quantizer hook helpers before this method is called.Args:block: Module already placed on the correct device(s) with act_maxdiff --git a/auto_round/algorithms/quantization/sign_round/quantizer.py b/auto_round/algorithms/quantization/sign_round/quantizer.pyindex e0859d260..2297d9d44 100644--- a/auto_round/algorithms/quantization/sign_round/quantizer.py+++ b/auto_round/algorithms/quantization/sign_round/quantizer.py@@ -36,7 +36,6 @@ from auto_round.logger import loggerfrom auto_round.utils import (check_to_quantized,compile_func,- convert_module_to_hp_if_necessary,get_module,htcore,is_auto_device_mapping,@@ -44,6 +43,7 @@ from auto_round.utils import (memory_monitor,mv_module_from_gpu,set_amax_for_all_moe_layers,+ set_module,to_device,)from auto_round.utils.device import (@@ -324,21 +324,39 @@ class SignRoundQuantizer(BaseQuantizers):return best_paramsdef quantize_layer_outside_block(- self, layer_name: str, input_ids: torch.Tensor, q_inputs: torch.Tensor = None, device: str = "cpu", **kwargs+ self,+ layer_name: str,+ input_ids: Optional[list[torch.Tensor]] = None,+ q_inputs: Optional[list[torch.Tensor]] = None,+ device: str = "cpu",+ dtype: Optional[torch.dtype] = None,):"""Quantize a specific layer of the model using the provided inputs.Args:layer_name (str): The name of the layer to quantize.- inputs (torch.Tensor): Input data for quantization.+ input_ids (list[torch.Tensor], optional): Input data for quantization.q_inputs (torch.Tensor, optional): Quantized input data. Defaults to None.device (torch.device, optional): The device to use for quantization. Defaults to torch.device("cpu").Returns:None"""+ if input_ids is None:+ logger.info(f"using rtn to quantize {layer_name}")+ if dtype is not None:+ layer = get_module(self.model, layer_name)+ set_module(self.model, layer_name, layer.to(dtype))+ self.quantize_layer_via_rtn(+ layer_name,+ disable_opt_rtn=getattr(self.config, "disable_opt_rtn", True),+ )+ return+logger.info(f"quantizing layer {layer_name}")layer = get_module(self.model, layer_name)+ if dtype is not None:+ layer = layer.to(dtype)if hasattr(layer, "tuning_device"):device = layer.tuning_device@@ -358,7 +376,7 @@ class SignRoundQuantizer(BaseQuantizers):static_attention_dtype,):tmp_inputs = q_inputs if q_inputs is not None else input_ids- hook_handles = self._register_act_max_hook(layer)+ hook_handles = self.register_calibration_hooks(layer)with torch.no_grad():for input in tmp_inputs:layer(input)diff --git a/auto_round/algorithms/quantization/sign_roundv2/quantizer.py b/auto_round/algorithms/quantization/sign_roundv2/quantizer.pyindex 145215dd1..b8ff6c9bb 100644--- a/auto_round/algorithms/quantization/sign_roundv2/quantizer.py+++ b/auto_round/algorithms/quantization/sign_roundv2/quantizer.py@@ -22,6 +22,7 @@ from torch import autocastfrom auto_round.algorithms.quantization.sign_round.config import SignRoundConfigfrom auto_round.algorithms.quantization.sign_round.quantizer import SignRoundQuantizer+from auto_round.algorithms.quantization.utils import register_imatrix_hooksfrom auto_round.data_type.gguf import (double_quant_tensor_sym_rtn,quant_tensor_gguf_asym_dq,@@ -360,26 +361,15 @@ class SignRoundV2Quantizer(SignRoundQuantizer):)return super()._get_loss(output_q, current_output, indices, mse_loss, device)- def _register_act_max_hook(self, model):- hook_handles = super()._register_act_max_hook(model)+ def register_calibration_hooks(self, model, *, act_max: bool = True, imatrix: bool = True):+ hook_handles = super().register_calibration_hooks(model, act_max=act_max, imatrix=imatrix)+ if not imatrix:+ return hook_handlesis_wint4aint4 = ("int4" in self.act_data_type or ("int" in self.act_data_type and self.act_bits == 4)) and ("int4" in self.data_type or ("int" in self.data_type and self.bits == 4))if is_wint4aint4:return hook_handles-- def get_imatrix_hook(module, input, output):- input = input[0] if isinstance(input, (tuple, list)) else input- flattened = input.reshape(-1, input.shape[-1]).to(torch.float32)- squared = torch.sum(torch.pow(flattened, 2), dim=0).to(torch.float32)-- if not hasattr(module, "imatrix"):- module.imatrix = squared- else:- module.imatrix += squared.to(module.imatrix.device)-- for _, module in model.named_modules():- if isinstance(module, self.supported_types) and check_to_quantized(module):- hook_handles.append(module.register_forward_hook(get_imatrix_hook))+ hook_handles.extend(register_imatrix_hooks(self, model))return hook_handlesdiff --git a/auto_round/algorithms/quantization/utils.py b/auto_round/algorithms/quantization/utils.pynew file mode 100644index 000000000..ac896183c--- /dev/null+++ b/auto_round/algorithms/quantization/utils.py@@ -0,0 +1,98 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++import torch++from auto_round.compressors.utils import check_need_act_calibration+from auto_round.data_type.utils import reshape_pad_tensor_by_group_size+from auto_round.utils import SUPPORTED_LAYER_TYPES, check_to_quantized+++def register_act_max_hooks(quantizer, model):++ def get_act_max_hook(module, input, output):+ if isinstance(input, (tuple, list)):+ input = input[0]+ if input.numel() == 0:+ return+ input, _, _ = reshape_pad_tensor_by_group_size(input, quantizer.act_group_size)+ act_max = torch.max(torch.abs(input), dim=-1).values+ if not hasattr(module, "act_max") or module.act_max.numel() == 0:+ module.act_max = act_max+ if quantizer.config.is_act_nv_fp:+ max_val = act_max.max()+ module.act_max = max_val.unsqueeze(0) if max_val.dim() == 0 else max_val+ else:+ act_max = act_max.to(module.act_max.device)+ if quantizer.config.is_act_nv_fp:+ max_val = torch.max(act_max.max(), module.act_max.max())+ module.act_max = max_val.unsqueeze(0) if max_val.dim() == 0 else max_val+ else:+ module.act_max = torch.max(act_max, module.act_max)++ hook_handles = []+ if isinstance(model, SUPPORTED_LAYER_TYPES):+ if (+ hasattr(model, "act_dynamic")+ and check_need_act_calibration(model.act_dynamic, model.act_data_type, model.act_bits)+ and check_to_quantized(model)+ ):+ hook_handles.append(model.register_forward_hook(get_act_max_hook))+ return hook_handles++ for name, module in model.named_modules():+ if (+ hasattr(module, "act_dynamic")+ and check_need_act_calibration(module.act_dynamic, module.act_data_type, module.act_bits)+ and check_to_quantized(module)+ ):+ hook_handles.append(module.register_forward_hook(get_act_max_hook))+ continue++ if name in quantizer.layer_config:+ config = quantizer.layer_config[name]+ act_dynamic = config.get("act_dynamic", True)+ act_data_type = config.get("act_data_type", None)+ act_bits = config.get("act_bits", 16)+ if (+ config["bits"] <= 8+ and check_need_act_calibration(act_dynamic, act_data_type, act_bits)+ and check_to_quantized(config)+ ):+ hook_handles.append(module.register_forward_hook(get_act_max_hook))+ continue+ return hook_handles+++def register_imatrix_hooks(quantizer, model, *, with_count: bool = False):++ def get_imatrix_hook(module, input, output):+ input = input[0] if isinstance(input, (tuple, list)) else input+ flattened = input.reshape(-1, input.shape[-1]).to(torch.float32)+ squared = torch.sum(torch.pow(flattened, 2), dim=0).to(torch.float32)++ if not hasattr(module, "imatrix"):+ module.imatrix = squared+ if with_count:+ module.imatrix_cnt = input.shape[0]+ else:+ module.imatrix += squared.to(module.imatrix.device)+ if with_count:+ module.imatrix_cnt += input.shape[0]++ hook_handles = []+ for _, module in model.named_modules():+ if isinstance(module, quantizer.supported_types) and check_to_quantized(module):+ hook_handles.append(module.register_forward_hook(get_imatrix_hook))+ return hook_handlesdiff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex 12b841ace..c2e2742c4 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -152,7 +152,13 @@ class AutoRound:local_args = {k: v for k, v in locals().items() if k not in cls.SKIP_ARGS}if extra_config is not None:- local_args.update({k: v for k, v in extra_config.to_dict().items() if k in local_args and v is not None})+ for key, value in extra_config.to_dict().items():+ if value is None:+ continue+ if key in local_args:+ local_args[key] = value+ else:+ kwargs[key] = valuefrom auto_round.compressors.entry import AutoRoundCompatiblediff --git a/auto_round/calibration/hooks.py b/auto_round/calibration/hooks.pyindex ff4c6ddd9..0d356ee9f 100644--- a/auto_round/calibration/hooks.py+++ b/auto_round/calibration/hooks.py@@ -105,6 +105,18 @@ def make_block_forward_func(state, name: str) -> Callable:else: # append cache inputsnew_data = post_process_cache_data(state.quantizer.batch_size, kwargs[key], key)if new_data is None: # shareable args or NoneType+ if key in state.model_context.shared_cache_keys:+ # Shared keys are normally the same across samples. However+ # in VLM visual encoders (e.g. Qwen2-VL) ``position_embeddings``+ # varies per image because each image has a different patch count.+ # Upgrade from shared (raw value) to per-sample list storage so+ # each sample gets its own positional embeddings.+ raw_new = to_device(kwargs[key], device=torch.device("cpu"))+ stored = state.inputs[name].get(key)+ if isinstance(stored, list):+ stored.append(raw_new)+ elif stored is not None:+ state.inputs[name][key] = [stored, raw_new]continuenew_data = to_device(new_data, device=torch.device("cpu"))if state.quantizer.batch_size <= 1:diff --git a/auto_round/calibration/mllm.py b/auto_round/calibration/mllm.pyindex 62dfd2448..0c403a54a 100644--- a/auto_round/calibration/mllm.py+++ b/auto_round/calibration/mllm.py@@ -44,7 +44,7 @@ class MLLMCalibrator(LLMCalibrator):from auto_round.compressors.mllm.dataset import get_mllm_dataloaderfrom auto_round.compressors.mllm.template import get_template- from auto_round.special_model_handler import MISTRAL_3_2_MODELS+ from auto_round.special_model_handler import MISTRAL_3_2_MODELS, NOT_SUPPORT_ONLY_TEXT_MODELSc = self.compressormc = c.model_context@@ -97,6 +97,13 @@ class MLLMCalibrator(LLMCalibrator):" switching to liuhaotian/llava_conv_58k")dataset = "liuhaotian/llava_conv_58k"+ elif dataset in CALIB_DATASETS and c.template_obj.model_type in NOT_SUPPORT_ONLY_TEXT_MODELS:+ logger.warning(+ f"{getattr(mc.model.config, 'model_type', c.template_obj.model_type)}"+ f" does not support for {dataset},"+ " will use liuhaotian/llava_conv_58k with default config as an alternative."+ )+ dataset = "liuhaotian/llava_conv_58k"(c.dataloader,c.batch_size,@@ -114,6 +121,7 @@ class MLLMCalibrator(LLMCalibrator):bs=bs,seed=c.seed,nsamples=nsamples,+ gradient_accumulate_steps=c.gradient_accumulate_steps,quant_nontext_module=c.quant_nontext_module,)else:diff --git a/auto_round/calibration/state.py b/auto_round/calibration/state.pyindex 85f28d67d..f4fdfc9f7 100644--- a/auto_round/calibration/state.py+++ b/auto_round/calibration/state.py@@ -19,7 +19,7 @@ This dataclass owns every per-run calibration field shared between- Cache state ``(inputs, to_cached_layers, last_cache_name, blocks_requiring_input_ids)``- Per-batch shape state ``(attention_mask, batch_dim)``-- Calibration parameters ``(batch_size, nsamples, seqlen, dataset, dataloader)``+- Calibration parameters ``(batch_size, gradient_accumulate_steps, nsamples, seqlen, dataset, dataloader)``Both the compressor and the quantizer hold a reference to the same``CalibrationState`` instance (wired in ``BaseCompressor._resolve_scheme``).@@ -61,6 +61,7 @@ class CalibrationState:# ── Calibration parameters ─────────────────────────────────────────────batch_size: int = 8+ gradient_accumulate_steps: int = 1nsamples: int = 128seqlen: int = 2048dataset: Any = None@@ -87,6 +88,7 @@ class CalibrationState:last_cache_name=getattr(compressor, "last_cache_name", None),blocks_requiring_input_ids=getattr(compressor, "blocks_requiring_input_ids", []) or [],batch_size=getattr(compressor, "batch_size", 8) or 8,+ gradient_accumulate_steps=getattr(compressor, "gradient_accumulate_steps", 1) or 1,nsamples=getattr(compressor, "nsamples", 128) or 128,seqlen=getattr(compressor, "seqlen", 2048) or 2048,dataset=getattr(compressor, "dataset", None),diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.pyindex af36c0324..c48618c4a 100644--- a/auto_round/compressors/base.py+++ b/auto_round/compressors/base.py@@ -172,6 +172,7 @@ class BaseCompressor(object):nsamples=nsamples if nsamples is not None else 128,seqlen=seqlen if seqlen is not None else 2048,batch_size=kwargs.pop("batch_size", 8),+ gradient_accumulate_steps=kwargs.pop("gradient_accumulate_steps", 1),)# ``dataset`` is not a named __init__ parameter – it arrives via@@ -213,6 +214,9 @@ class BaseCompressor(object):# Extra/legacy kwargs for backward compatibility# Major version releases may pack them with extra configuration options+ kwargs.pop("iters", None)+ kwargs.pop("enable_alg_ext", None)+ kwargs.pop("vlm", None)amp = kwargs.pop("amp", True)nblocks = kwargs.pop("nblocks", 1)disable_deterministic_algorithms = kwargs.pop("disable_deterministic_algorithms", True)@@ -224,6 +228,9 @@ class BaseCompressor(object):model_dtype = kwargs.pop("model_dtype", None)trust_remote_code = kwargs.pop("trust_remote_code") if "trust_remote_code" in kwargs else Truequant_nontext_module = kwargs.pop("quant_nontext_module", False)+ device = kwargs.pop("device", None)+ if device is not None:+ logger.warning("`device` is deprecated, please use `device_map` instead")self.static_attention_dtype = kwargs.pop("static_attention_dtype", None)# Attention static dtype@@ -258,10 +265,6 @@ class BaseCompressor(object):# instead of MATH (avoids ~10x peak-VRAM blow-up during block tuning).patch_xpu_sdpa_drop_causal_mask()- device = kwargs.pop("device", None)- if device is not None:- logger.warning("`device` is deprecated, please use `device_map` instead")-# Tuning hyperparametersself.seed = seedset_seed(self.seed)@@ -327,7 +330,7 @@ class BaseCompressor(object):# Apply torch compile adjustments eagerly so that ar.enable_torch_compile# reflects the correct value immediately after construction (not only after post_init).- self._adjust_torch_compile(enable_torch_compile)+ self._precheck_torch_compile(enable_torch_compile)self.compress_context.enable_torch_compile = self.enable_torch_compile# ``self._calibration_state`` was created at the top of __init__ so@@ -620,10 +623,8 @@ class BaseCompressor(object):def diffusion(self):return self.model_context.is_diffusion- def _adjust_torch_compile(self, enable_torch_compile: bool) -> None:- """Sets the torch compile configuration for the tuning."""- self.enable_torch_compile = enable_torch_compile-+ def _get_torch_compile_guard_state(self) -> tuple[bool, bool, int]:+ """Return raw dtype state used by torch.compile guard rules."""# Determine fp8 / nvfp4 intent from raw config before scheme resolution.cfg = self.quantize_configraw_scheme = self.scheme if isinstance(self.scheme, str) else ""@@ -641,6 +642,11 @@ class BaseCompressor(object):)act_bits = getattr(cfg, "act_bits", 16) or 16+ return is_raw_fp8, is_raw_nv_fp, act_bits++ def _maybe_log_torch_compile_default_hint(self) -> None:+ """Log the default torch.compile hint once final config state is available."""+ is_raw_fp8, _, act_bits = self._get_torch_compile_guard_state()if (not self.enable_torch_compileand TORCH_VERSION_AT_LEAST_2_6@@ -654,6 +660,13 @@ class BaseCompressor(object):"'enable_torch_compile' is set to `False` by default. ""Enabling it can reduce tuning cost by 20%, but it might throw an exception.",)++ def _apply_torch_compile_constraints(self, enable_torch_compile: bool) -> None:+ """Apply torch.compile disabling rules for the current compressor state."""+ self.enable_torch_compile = enable_torch_compile+ cfg = self.quantize_config+ is_raw_fp8, is_raw_nv_fp, _ = self._get_torch_compile_guard_state()+# On HPU, we rely on torch.compile to speed up the model execution.if self.enable_torch_compile and is_raw_fp8 and not is_hpex_available():self.enable_torch_compile = False@@ -670,6 +683,22 @@ class BaseCompressor(object):# "reset enable_torch_compile to `False` as super_group_size is set for algorithm extension"# )+ def _precheck_torch_compile(self, enable_torch_compile: bool) -> None:+ """Apply early torch.compile adjustments before scheme resolution.++ This runs during ``__init__`` so the compressor exposes a sensible+ ``enable_torch_compile`` value immediately after construction, even+ though scheme resolution has not completed yet.+ """+ self._apply_torch_compile_constraints(enable_torch_compile)++ def _finalize_torch_compile(self) -> None:+ """Re-evaluate torch.compile after scheme resolution with final attrs."""+ requested_enable_torch_compile = self.enable_torch_compile+ self._apply_torch_compile_constraints(requested_enable_torch_compile)+ if not requested_enable_torch_compile:+ self._maybe_log_torch_compile_default_hint()+def _get_calibration_dataset(self) -> str:"""Resolve calibration dataset: self.dataset > AutoScheme.dataset > default."""dataset = self._calibration_state.dataset@@ -1004,8 +1033,8 @@ class BaseCompressor(object):Preconditions:- Phase 4 complete: ``layer_config`` is built and``has_qlayer_outside_block`` is known.- - ``self.quantize_config.data_type`` is the final resolved value- (needed by :meth:`_adjust_torch_compile`).+ - ``self.quantize_config.data_type`` is the final resolved value+ (needed by :meth:`_finalize_torch_compile`).Work performed:- Applies the device map via :func:`~auto_round.utils.device.set_non_auto_device_map`.@@ -1024,7 +1053,7 @@ class BaseCompressor(object):"""set_non_auto_device_map(self.model_context.model, self.compress_context.device_map)# Re-evaluate torch.compile eligibility now that data_type is resolved.- self._adjust_torch_compile(self.enable_torch_compile)+ self._finalize_torch_compile()self.compress_context.enable_torch_compile = self.enable_torch_compileif self.compress_context.low_cpu_mem_usage:self._offloader.reset()@@ -1115,6 +1144,15 @@ class BaseCompressor(object):def batch_size(self, value: int) -> None:self._calibration_state.batch_size = value+ @property+ def gradient_accumulate_steps(self) -> int:+ return self._calibration_state.gradient_accumulate_steps++ @gradient_accumulate_steps.setter+ def gradient_accumulate_steps(self, value: int) -> None:+ if value is not None:+ self._calibration_state.gradient_accumulate_steps = value+@propertydef nsamples(self) -> int:return self._calibration_state.nsamplesdiff --git a/auto_round/compressors/data_driven.py b/auto_round/compressors/data_driven.pyindex d3cc72e41..c78e11d6e 100644--- a/auto_round/compressors/data_driven.py+++ b/auto_round/compressors/data_driven.py@@ -59,7 +59,6 @@ from auto_round.utils import (memory_monitor,mv_module_from_gpu,set_amax_for_all_moe_layers,- set_module,to_device,to_dtype,wrap_block_forward_positional_to_kwargs,@@ -68,7 +67,7 @@ from auto_round.utils.device import (_force_trim_malloc,parse_available_devices,)-from auto_round.wrapper import WrapperLinear, WrapperMultiblock+from auto_round.wrapper import WrapperMultiblockclass DataDrivenCompressor(BaseCompressor):@@ -300,9 +299,10 @@ class DataDrivenCompressor(BaseCompressor):"""from auto_round.calibration.state import CalibrationState- assert not self.mllm and not self.diffusion, (- f"Currently, {self.__class__.__name__} does not support quantize_block " "for MLLM / diffusion models."- )+ if self.diffusion:+ raise NotImplementedError(+ f"Currently, {self.__class__.__name__} does not support quantize_block for diffusion models."+ )# Ensure post_init has been called (sets up model_context, compress_context,# quantizer, layer_config, etc.).@@ -365,13 +365,13 @@ class DataDrivenCompressor(BaseCompressor):# ── Infrastructure: collect reference output and act_max ──────────────bs = self.quantizer.batch_size * self.quantizer.infer_bs_coeffif q_input is None:- hook_handles = self.quantizer._register_act_max_hook(block)+ hook_handles = self.quantizer.register_calibration_hooks(block)reference_output = self.quantizer._get_block_outputs(block, input_ids, input_others, bs)for h in hook_handles:h.remove()else:reference_output = self.quantizer._get_block_outputs(block, input_ids, input_others, bs)- hook_handles = self.quantizer._register_act_max_hook(block)+ hook_handles = self.quantizer.register_calibration_hooks(block)if hook_handles:self.quantizer._get_block_outputs(block, q_input, input_others, bs, save_output=False)for h in hook_handles:@@ -514,7 +514,7 @@ class DataDrivenCompressor(BaseCompressor):# ── Infrastructure: collect reference output and act_max ──────────bs = self.quantizer.batch_size * self.quantizer.infer_bs_coeffif q_input is None:- hook_handles = self.quantizer._register_act_max_hook(m)+ hook_handles = self.quantizer.register_calibration_hooks(m)reference_output = self.quantizer._get_block_outputs(m, input_ids, input_others, bs, device_override=loss_device)@@ -524,7 +524,7 @@ class DataDrivenCompressor(BaseCompressor):reference_output = self.quantizer._get_block_outputs(m, input_ids, input_others, bs, device_override=loss_device)- hook_handles = self.quantizer._register_act_max_hook(m)+ hook_handles = self.quantizer.register_calibration_hooks(m)if hook_handles:self.quantizer._get_block_outputs(m, q_input, input_others, bs, save_output=False, device_override=loss_device@@ -797,28 +797,12 @@ class DataDrivenCompressor(BaseCompressor):)layer_names.remove(layer_name)continue- logger.info(f"using rtn to quantize {layer_name}")- from auto_round.data_type import QUANT_FUNC_WITH_DTYPE-- layer = get_module(self.model, layer_name)- layer = layer.to(self.compress_context.device)- layer = convert_module_to_hp_if_necessary(- layer, self.model_context.amp_dtype, self.compress_context.device- )- set_module(self.model, layer_name, layer)-- wrapper_layer = WrapperLinear(- layer,- enable_round_tuning=False,- enable_minmax_tuning=False,- enable_norm_bias_tuning=False,- enable_torch_compile=self.enable_torch_compile,+ self.quantizer.quantize_layer_outside_block(+ layer_name,+ input_ids=None,device=self.compress_context.device,disable_opt_rtn=getattr(self, "disable_opt_rtn", False),)- new_layer = wrapper_layer.unwrapper({})- set_module(self.model, layer_name, new_layer)- layer.cpu()layer_names.remove(layer_name)if len(layer_names) == 0:memory_monitor.update()@@ -1020,7 +1004,7 @@ class CalibratedRTNCompressor(DataDrivenCompressor):block = block.to(self.compress_context.device)# ── Infrastructure: register act_max hook and run forward pass ──- hook_handles = self.quantizer._register_act_max_hook(block)+ hook_handles = self.quantizer.register_calibration_hooks(block, imatrix=False)input_ids = self.quantizer._get_block_outputs(block,input_ids,@@ -1077,14 +1061,14 @@ class CalibratedRTNCompressor(DataDrivenCompressor):def _quant_rtn_with_imatrix(self) -> None:"""Performs RTN quantization using input activation statistics (imatrix).- This method accumulates per-channel second-moment activation statistics (imatrix)- via forward hooks and uses them to perform RTN quantization. If CUDA memory runs out,- it falls back to CPU-based blockwise quantization.+ OptimizedRTNQuantizer owns imatrix hook registration. This method only+ enables the quantizer-side collection path and keeps the OOM fallback.Returns:None"""logger.info("start to compute imatrix")+ self.quantizer.enable_imatrix = True# Dataloader resolution is owned by ``CalibrationState``.self._calibration_state.ensure_dataloader(self.model_context, self.seed)@@ -1095,30 +1079,7 @@ class CalibratedRTNCompressor(DataDrivenCompressor):if hasattr(model, "hf_device_map") and len(model.hf_device_map) > 1:dispatch_model(model, model.hf_device_map)- def register_act_hook(model):- """Registers hooks to accumulate activation squared norms into `imatrix`."""-- def get_imatrix_hook(module, input, output):- input = input[0] if isinstance(input, (tuple, list)) else input- flattened = input.reshape(-1, input.shape[-1]).to(torch.float32)- squared = torch.sum(torch.pow(flattened, 2), dim=0).to(torch.float32)-- if not hasattr(module, "imatrix"):- module.imatrix = squared- module.imatrix_cnt = input.shape[0]- else:- module.imatrix += squared.to(module.imatrix.device)- module.imatrix_cnt += input.shape[0]-- hook_handles = []- for name, module in model.named_modules():- if type(module) in SUPPORTED_LAYER_TYPES and check_to_quantized(module):- hook = module.register_forward_hook(get_imatrix_hook)- hook_handles.append(hook)- return hook_handles-- hooks = register_act_hook(model)-+ hooks = self.quantizer.register_calibration_hooks(model, act_max=False)try:if hasattr(model, "hf_device_map") and len(model.hf_device_map) > 1:import accelerate@@ -1150,9 +1111,9 @@ class CalibratedRTNCompressor(DataDrivenCompressor):except Exception as e:raisefinally:- # Always remove hooksfor hook in hooks:hook.remove()+ self.quantizer.enable_imatrix = Falsedef quantize(self):"""Quantize all modules in the model using RTN (Round-To-Nearest) strategy.diff --git a/auto_round/compressors/entry.py b/auto_round/compressors/entry.pyindex 23b76cf55..e7f3fd67b 100644--- a/auto_round/compressors/entry.py+++ b/auto_round/compressors/entry.py@@ -189,6 +189,8 @@ class AutoRound(object):scheme="W4A16",low_gpu_mem_usage: bool = False,device_map: Union[str, torch.device, int, dict] = 0,+ iters: int = None,+ gradient_accumulate_steps: int = 1,enable_torch_compile: bool = False,seed: int = 42,low_cpu_mem_usage: bool = True,@@ -232,6 +234,8 @@ class AutoRound(object):scheme=scheme,low_gpu_mem_usage=low_gpu_mem_usage,device_map=device_map,+ iters=iters,+ gradient_accumulate_steps=gradient_accumulate_steps,enable_torch_compile=enable_torch_compile,seed=seed,low_cpu_mem_usage=low_cpu_mem_usage,@@ -311,6 +315,7 @@ class AutoRound(object):from auto_round.auto_scheme.gen_auto_scheme import AutoScheme as _AutoSchemeis_auto_scheme = isinstance(scheme, _AutoScheme)+ quant_config.enable_imatrix = enable_imatrixif enable_imatrix or needs_act_calib or is_auto_scheme:quant_config._alg_cls = "OptimizedRTNQuantizer"@@ -429,6 +434,12 @@ class AutoRoundCompatible:from auto_round.utils import is_diffusion_model, is_mllm_modelfrom auto_round.utils.model import is_model_free_route+ device = kwargs.pop("device", None)+ if device is not None:+ logger.warning_once("`device` is deprecated, please use `device_map` instead")+ if device_map in (None, 0):+ device_map = device+# ---- Model-free fast-path detection --------------------------------if is_model_free_route(model, scheme, iters, kwargs.get("disable_opt_rtn"), kwargs):from auto_round.compressors.model_free import ModelFreeCompressor@@ -463,6 +474,12 @@ class AutoRoundCompatible:act_sym = kwargs.pop("act_sym", None)act_data_type = kwargs.pop("act_data_type", None)act_dynamic = kwargs.pop("act_dynamic", None)+ enable_opt_rtn = kwargs.pop("enable_opt_rtn", None)+ lr = kwargs.pop("lr", None)+ minmax_lr = kwargs.pop("minmax_lr", None)+ enable_minmax_tuning = kwargs.pop("enable_minmax_tuning", True)+ enable_norm_bias_tuning = kwargs.pop("enable_norm_bias_tuning", False)+ enable_quanted_input = kwargs.pop("enable_quanted_input", True)# Decide which algorithm to useif iters == 0:@@ -479,16 +496,11 @@ class AutoRoundCompatible:act_data_type=act_data_type,act_dynamic=act_dynamic,disable_opt_rtn=disable_opt_rtn,+ enable_opt_rtn=enable_opt_rtn,**common_config_kwargs,)else:# AutoRoundCompatible mode- lr = kwargs.pop("lr", None)- minmax_lr = kwargs.pop("minmax_lr", None)- enable_minmax_tuning = kwargs.pop("enable_minmax_tuning", True)- enable_norm_bias_tuning = kwargs.pop("enable_norm_bias_tuning", False)- enable_quanted_input = kwargs.pop("enable_quanted_input", True)-config = SignRoundConfig(iters=iters,gradient_accumulate_steps=gradient_accumulate_steps,@@ -559,6 +571,7 @@ class AutoRoundCompatible:scheme=scheme,dataset=dataset,iters=iters,+ gradient_accumulate_steps=gradient_accumulate_steps,low_gpu_mem_usage=low_gpu_mem_usage,device_map=device_map,enable_torch_compile=enable_torch_compile,diff --git a/auto_round/compressors/zero_shot.py b/auto_round/compressors/zero_shot.pyindex 1cd2ced4c..3b471c6a2 100644--- a/auto_round/compressors/zero_shot.py+++ b/auto_round/compressors/zero_shot.py@@ -65,7 +65,6 @@ class ZeroShotCompressor(BaseCompressor):device_map=device_map,low_gpu_mem_usage=low_gpu_mem_usage,enable_torch_compile=enable_torch_compile,- enable_alg_ext=enable_alg_ext,seed=seed,low_cpu_mem_usage=low_cpu_mem_usage,**kwargs,diff --git a/test/test_cpu/core/test_llmc_quantize_block.py b/test/test_cpu/core/test_llmc_quantize_block.pynew file mode 100644index 000000000..28cae1345--- /dev/null+++ b/test/test_cpu/core/test_llmc_quantize_block.py@@ -0,0 +1,64 @@+from types import SimpleNamespace++import torch++from auto_round.compressors.data_driven import DataDrivenCompressor+++class _QuantizerStub:+ batch_size = 1+ infer_bs_coeff = 1+ enable_quanted_input = False+ act_data_type = ""++ def register_calibration_hooks(self, block, *, act_max=True, imatrix=True):+ return []++ def _get_block_outputs(self, block, input_ids, input_others, bs, save_output=True):+ return torch.ones(1, 1)++ def quantize_block(+ self, block, input_ids, input_others, reference_output, loss_device=None, mid_iter_mem_check=False+ ):+ return None+++def test_llmc_quantize_block_allows_mllm(monkeypatch):+ compressor = object.__new__(DataDrivenCompressor)+ compressor.model_context = SimpleNamespace(is_mllm=True, is_diffusion=False, amp_dtype=torch.float32)+ compressor._post_init_done = True+ compressor._calibration_state = SimpleNamespace(inputs={})+ compressor.compress_context = SimpleNamespace(+ device_map="cpu",+ device_list=["cpu"],+ low_gpu_mem_usage=False,+ )+ compressor.quantizer = _QuantizerStub()+ compressor.quant_block_list = [["decoder.layers.0"]]++ def _normalize(_inputs):+ compressor.inputs = {"decoder.layers.0": [(torch.ones(1, 1), {})]}++ def _preprocess(_block_inputs, _input_name="hidden_states"):+ assert compressor.model_context.is_mllm is False+ return torch.ones(1, 1), {}++ compressor.normalize_decoding_layer_inputs_ = _normalize+ compressor._preprocess_block_inputs = _preprocess++ monkeypatch.setattr("auto_round.compressors.data_driven.materialize_model_", lambda block: None)+ monkeypatch.setattr(+ "auto_round.compressors.data_driven.convert_module_to_hp_if_necessary",+ lambda block, amp_dtype, device: None,+ )+ monkeypatch.setattr("auto_round.compressors.data_driven.mv_module_from_gpu", lambda block: None)+ monkeypatch.setattr("auto_round.compressors.data_driven.clear_memory", lambda *args, **kwargs: None)+ monkeypatch.setattr("auto_round.compressors.data_driven.is_nv_fp", lambda *_args, **_kwargs: False)+ monkeypatch.setattr("auto_round.compressors.data_driven.is_static_wfp8afp8", lambda *_args, **_kwargs: False)++ block = torch.nn.Linear(1, 1)+ q_outputs, reference_output = compressor.quantize_block(block, inputs=[((torch.ones(1, 1), {}),)])++ assert q_outputs is None+ assert torch.equal(reference_output, torch.ones(1, 1))+ assert compressor.model_context.is_mllm is True