Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

Fix few merge errors (#1817)

Heng Guo706ac7e39861
diff --git a/auto_round/algorithms/quantization/base.py b/auto_round/algorithms/quantization/base.py
index b0e0cbf6b..765991844 100644
--- a/auto_round/algorithms/quantization/base.py
+++ b/auto_round/algorithms/quantization/base.py
@@ -19,12 +19,12 @@ from typing import Union
import torch
from auto_round.algorithms.quantization.config import QuantizationConfig
+from auto_round.algorithms.quantization.utils import register_act_max_hooks
from auto_round.compressors.utils import (
block_forward,
- check_need_act_calibration,
+ immediate_pack,
)
from auto_round.data_type import QUANT_FUNC_WITH_DTYPE
-from auto_round.data_type.utils import reshape_pad_tensor_by_group_size
from auto_round.logger import logger
from auto_round.utils import (
INNER_SUPPORTED_LAYER_TYPES,
@@ -32,7 +32,11 @@ from auto_round.utils import (
check_to_quantized,
clear_memory,
compile_func,
+ convert_module_to_hp_if_necessary,
+ get_module,
+ set_module,
)
+from auto_round.wrapper import WrapperLinear
class BaseQuantizers:
@@ -172,66 +176,8 @@ class BaseQuantizers:
return getattr(self.model_context, "amp_dtype", torch.float32)
- def _register_act_max_hook(self, model):
-
- def get_act_max_hook(module, input, output):
- if isinstance(input, (tuple, list)):
- input = input[0]
- if input.numel() == 0:
- return # as no needs for act_max update
- input, _, _ = reshape_pad_tensor_by_group_size(input, self.act_group_size)
- act_max = torch.max(torch.abs(input), dim=-1).values
- if not hasattr(module, "act_max") or module.act_max.numel() == 0:
- module.act_max = act_max
- if self.config.is_act_nv_fp: ## for nvfp per-tensor input_global_scale calculation usage
- max_val = act_max.max()
- module.act_max = max_val.unsqueeze(0) if max_val.dim() == 0 else max_val
- else:
- act_max = act_max.to(module.act_max.device)
- if self.config.is_act_nv_fp: ## for nvfp per-tensor input_global_scale calculation usage
- max_val = torch.max(act_max.max(), module.act_max.max())
- module.act_max = max_val.unsqueeze(0) if max_val.dim() == 0 else max_val
- else:
- module.act_max = torch.max(act_max, module.act_max)
-
- hook_handles = []
- # for single layers out of blocks, like lm_head
- if isinstance(model, SUPPORTED_LAYER_TYPES):
- m = model
- if (
- hasattr(m, "act_dynamic")
- and check_need_act_calibration(m.act_dynamic, m.act_data_type, m.act_bits)
- and check_to_quantized(m)
- ):
- hook = m.register_forward_hook(get_act_max_hook)
- hook_handles.append(hook)
- return hook_handles
-
- for n, m in model.named_modules():
- if (
- hasattr(m, "act_dynamic")
- and check_need_act_calibration(m.act_dynamic, m.act_data_type, m.act_bits)
- and check_to_quantized(m)
- ):
- hook = m.register_forward_hook(get_act_max_hook)
- hook_handles.append(hook)
- continue
-
- # for whole model, RTN
- if n in self.layer_config:
- config = self.layer_config[n]
- act_dynamic = config.get("act_dynamic", True)
- act_data_type = config.get("act_data_type", None)
- act_bits = config.get("act_bits", 16)
- if (
- config["bits"] <= 8
- and check_need_act_calibration(act_dynamic, act_data_type, act_bits)
- and check_to_quantized(config)
- ):
- hook = m.register_forward_hook(get_act_max_hook)
- hook_handles.append(hook)
- continue
- return hook_handles
+ def register_calibration_hooks(self, model, *, act_max: bool = True, imatrix: bool = True):
+ return register_act_max_hooks(self, model) if act_max else []
@torch.inference_mode()
def _quantize_embedding_layer(self):
@@ -343,6 +289,94 @@ class BaseQuantizers:
"""
raise NotImplementedError("quantize_block must be implemented in subclasses of BaseQuantizers")
+ @torch.no_grad()
+ def quantize_layer_via_rtn(self, layer_name: str, disable_opt_rtn: bool | None = None) -> None:
+ """Quantize one layer with RTN and handle optional immediate pack/save."""
+ layer = get_module(self.model, layer_name)
+ layer = convert_module_to_hp_if_necessary(layer, self.model_context.amp_dtype, self.compress_context.device)
+ set_module(self.model, layer_name, layer)
+ tuning_device = layer.tuning_device if hasattr(layer, "tuning_device") else self.compress_context.device
+ if (
+ self.compress_context.is_immediate_packing
+ and self.compress_context.formats[0].is_gguf()
+ and not getattr(self.config, "disable_opt_rtn", False)
+ ):
+ layer = layer.to(tuning_device)
+ layer.scale = None
+ layer.zp = None
+ else:
+ try:
+ if disable_opt_rtn is None:
+ disable_opt_rtn = bool(getattr(self.config, "disable_opt_rtn", False))
+ if (
+ not disable_opt_rtn
+ and getattr(self.config, "orig_disable_opt_rtn", None) is None
+ and self.model_context.is_moe_model
+ and "expert" in layer.global_name
+ and "shared_expert" not in layer.global_name
+ and self.config.super_bits is None
+ ):
+ disable_opt_rtn = True
+ logger.warning_once(
+ "MoE layer detected: optimized RTN is disabled for efficiency. "
+ "Use `--enable_opt_rtn` to force-enable it for MoE layers."
+ )
+ layer = layer.to(tuning_device)
+ layer = WrapperLinear(
+ layer,
+ device=tuning_device,
+ enable_minmax_tuning=False,
+ enable_norm_bias_tuning=False,
+ enable_round_tuning=False,
+ enable_torch_compile=self.compress_context.enable_torch_compile,
+ disable_opt_rtn=disable_opt_rtn,
+ iters=0,
+ )
+ layer = layer.unwrapper({})
+ except torch.OutOfMemoryError:
+ cuda_error_msg = traceback.format_exc()
+ layer = layer.orig_layer if hasattr(layer, "orig_layer") else layer
+ try:
+ logger.error(cuda_error_msg)
+ logger.warning("falling back to CPU.")
+ layer.to("cpu")
+ layer = WrapperLinear(
+ layer,
+ enable_minmax_tuning=False,
+ enable_norm_bias_tuning=False,
+ enable_round_tuning=False,
+ enable_torch_compile=self.compress_context.enable_torch_compile,
+ iters=0,
+ )
+ layer = layer.unwrapper({})
+ except Exception:
+ raise
+
+ set_module(self.model, layer_name, layer)
+ self._immediate_pack_and_save_module(layer_name)
+
+ def _immediate_pack_and_save_module(self, module_name):
+ from auto_round.compressors.shard_writer import ShardWriter
+
+ shard_writer = ShardWriter.get_shard_writer()
+ to_cpu = self.compress_context.low_gpu_mem_usage
+ module = get_module(self.model, module_name)
+ if self.compress_context.is_immediate_packing:
+ immediate_pack(module_name, self.layer_config)
+ if to_cpu:
+ module = module.to("cpu")
+ packed_module = get_module(self.model, module_name)
+ set_module(self.model, module_name, packed_module.to("cpu"))
+ else:
+ if to_cpu:
+ module = module.to("cpu")
+ set_module(self.model, module_name, module)
+ if self.compress_context.is_immediate_saving:
+ module = get_module(self.model, module_name)
+ module.to("cpu")
+ shard_writer.write(module, module_name, False)
+ module.to("meta")
+
def quantize_layer(self, layer_name: str, **kwargs):
"""Quantizes a single layer of the model.
@@ -352,14 +386,19 @@ class BaseQuantizers:
"""
raise NotImplementedError("quantize_layer must be implemented in subclasses of BaseQuantizers")
- def quantize_layer_outside_block(self, layer_name: str, **kwargs):
+ def quantize_layer_outside_block(self, layer_name: str, input_ids=None, **kwargs):
"""Quantizes a single layer of the model outside of a block.
Args:
layer_name (str): The name of the layer to quantize. The layer module is
retrieved internally via get_module(model, layer_name).
+ input_ids: Optional calibration inputs for data-driven outside-layer quantization.
"""
- raise NotImplementedError("quantize_layer_outside_block must be implemented in subclasses of BaseQuantizers")
+ dtype = kwargs.pop("dtype", None)
+ if dtype is not None:
+ layer = get_module(self.model, layer_name)
+ set_module(self.model, layer_name, layer.to(dtype))
+ self.quantize_layer_via_rtn(layer_name, **kwargs)
@torch.no_grad()
def _get_block_outputs(
@@ -543,9 +582,18 @@ class BaseQuantizers:
# Shared keys are stored once (not per-sample), often wrapped in a
# 1-element list by the caching hook. Unwrap so the model receives
# the raw value (e.g. (cos, sin) tuple, not [(cos, sin)]).
+ # Exception: if the hook detected that the "shared" key actually varies
+ # per sample (e.g. position_embeddings in a VLM visual encoder), it
+ # upgrades the storage to a per-sample list with >1 elements.
val = input_others[key]
if isinstance(val, list) and len(val) == 1:
current_input_others[key] = val[0]
+ elif isinstance(val, list) and len(val) > 1:
+ # Per-sample storage for a nominally-shared key that varies across
+ # calibration samples (e.g. position_embeddings in Qwen2-VL visual
+ # encoder blocks where each image has a different patch count).
+ idx = int(indices[0]) if len(indices) == 1 else 0
+ current_input_others[key] = val[idx] if idx < len(val) else val[0]
else:
current_input_others[key] = val
elif not isinstance(input_others[key], (str, bool, type(None))):
diff --git a/auto_round/algorithms/quantization/rtn/quantizer.py b/auto_round/algorithms/quantization/rtn/quantizer.py
index af8312f26..d1c2c0779 100644
--- a/auto_round/algorithms/quantization/rtn/quantizer.py
+++ b/auto_round/algorithms/quantization/rtn/quantizer.py
@@ -11,7 +11,6 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-import traceback
from collections import defaultdict
from typing import Any, Callable, Optional, Union
@@ -21,7 +20,7 @@ import torch
from auto_round.algorithms.quantization.base import BaseQuantizers
from auto_round.algorithms.quantization.rtn.config import RTNConfig
from auto_round.algorithms.quantization.sign_round.quantizer import SignRoundQuantizer
-from auto_round.compressors.shard_writer import ShardWriter
+from auto_round.algorithms.quantization.utils import register_imatrix_hooks
from auto_round.compressors.utils import (
IndexSampler,
block_forward,
@@ -29,7 +28,6 @@ from auto_round.compressors.utils import (
check_skippable_keywords,
collect_best_params,
get_shared_keys,
- immediate_pack,
infer_bits_by_data_type,
init_cache,
reset_params,
@@ -39,7 +37,6 @@ from auto_round.data_type.utils import update_block_global_scale_if_needed
from auto_round.logger import logger
from auto_round.utils import (
check_to_quantized,
- convert_module_to_hp_if_necessary,
get_lm_head_name,
get_module,
htcore,
@@ -56,7 +53,7 @@ from auto_round.utils.device import (
set_auto_device_map_for_block_with_tuning,
set_non_auto_device_map,
)
-from auto_round.wrapper import WrapperLinear, WrapperMultiblock, unwrapper_block, unwrapper_layer, wrapper_block
+from auto_round.wrapper import WrapperMultiblock, unwrapper_block, unwrapper_layer, wrapper_block
class RTNQuantizer(BaseQuantizers):
@@ -100,110 +97,10 @@ class RTNQuantizer(BaseQuantizers):
@torch.no_grad()
def quantize_layer(self, name: str, dtype: torch.dtype = None) -> None:
- """Quantizes a layer using RTN (Round-To-Nearest) if available.
-
- This function attempts to quantize a layer by switching its data type to a
- `rtn_*` version if supported, then wraps and unwraps the module to apply
- quantization. If GPU memory is insufficient, it falls back to CPU.
-
- If packing is enabled (`immediate_packing`), the function will also export
- the quantized layer to the appropriate backend format.
-
- Args:
- name (str): Name of the layer to quantize.
-
- Raises:
- RuntimeError: If quantization fails for reasons unrelated to memory.
- """
-
- m = get_module(self.model, name)
if dtype is not None:
- m = m.to(dtype)
-
- m = convert_module_to_hp_if_necessary(m, self.model_context.amp_dtype, self.compress_context.device)
- set_module(self.model, name, m)
- tuning_device = m.tuning_device if hasattr(m, "tuning_device") else self.compress_context.device
- # Step 1: let gguf merge layers or rename module first and we will handle the RTN is gguf specific logic
- if (
- self.compress_context.is_immediate_packing
- and self.compress_context.formats[0].is_gguf()
- and not getattr(self.config, "disable_opt_rtn", False)
- ):
- m = m.to(tuning_device)
- m.scale = None
- m.zp = None
- else:
- try:
- disable_opt_rtn = bool(getattr(self.config, "disable_opt_rtn", False))
- if (
- not disable_opt_rtn
- and self.config.orig_disable_opt_rtn is None
- and self.model_context.is_moe_model
- and "expert" in m.global_name
- and "shared_expert" not in m.global_name
- and self.config.super_bits is None # GGUF still uses the optimized RTN for MoE layers
- ):
- disable_opt_rtn = True
- logger.warning_once(
- "MoE layer detected: optimized RTN is disabled for efficiency. "
- "Use `--enable_opt_rtn` to force-enable it for MoE layers."
- )
- m = m.to(tuning_device)
- m = WrapperLinear(
- m,
- device=tuning_device,
- enable_minmax_tuning=False,
- enable_norm_bias_tuning=False,
- enable_round_tuning=False,
- enable_torch_compile=self.compress_context.enable_torch_compile,
- disable_opt_rtn=disable_opt_rtn,
- iters=0,
- )
- m = m.unwrapper({})
- except torch.OutOfMemoryError:
- cuda_error_msg = traceback.format_exc()
- m = m.orig_layer if hasattr(m, "orig_layer") else m
- try:
- logger.error(cuda_error_msg)
- logger.warning("falling back to CPU.")
- m.to("cpu")
- m = WrapperLinear(
- m,
- enable_minmax_tuning=False,
- enable_norm_bias_tuning=False,
- enable_round_tuning=False,
- enable_torch_compile=self.compress_context.enable_torch_compile,
- iters=0,
- )
- m = m.unwrapper({})
- except Exception as e:
- raise
-
- set_module(self.model, name, m)
- self._immediate_pack_and_save_module(name)
-
- def _immediate_pack_and_save_module(self, module_name):
- shard_writer = ShardWriter.get_shard_writer()
- to_cpu = self.compress_context.low_gpu_mem_usage
- module = get_module(self.model, module_name)
- if self.compress_context.is_immediate_packing: # For gguf, packing conducts on block level
- immediate_pack(module_name, self.layer_config)
- if to_cpu:
- module = module.to("cpu")
- packed_module = get_module(self.model, module_name)
- set_module(self.model, module_name, packed_module.to("cpu"))
- else:
- if to_cpu:
- module = module.to("cpu")
- set_module(self.model, module_name, module)
- if self.compress_context.is_immediate_saving:
- module = get_module(self.model, module_name)
- module.to("cpu")
- shard_writer.write(module, module_name, False)
- # Free RAM immediately: the data is now in the shard-writer buffer
- # (and will be flushed to disk). Keeping it also in the model tree
- # causes linear RAM growth for large models.
- module.to("meta")
+ layer = get_module(self.model, name)
+ set_module(self.model, name, layer.to(dtype))
+ self.quantize_layer_via_rtn(name)
class OptimizedRTNQuantizer(RTNQuantizer):
@@ -213,11 +110,15 @@ class OptimizedRTNQuantizer(RTNQuantizer):
self.data_type = config.data_type
self.group_size = config.group_size
self.infer_bs_coeff = config.infer_bs_coeff
+ self.enable_imatrix = getattr(config, "enable_imatrix", False)
self.enable_alg_ext = True
- def quantize_layer_outside_block(self, *args, **kwargs):
- return self.quantize_layer(*args, **kwargs)
+ def register_calibration_hooks(self, model, *, act_max: bool = True, imatrix: bool = True):
+ hook_handles = super().register_calibration_hooks(model, act_max=act_max, imatrix=imatrix)
+ if imatrix and self.enable_imatrix:
+ hook_handles.extend(register_imatrix_hooks(self, model, with_count=True))
+ return hook_handles
@torch.no_grad()
def quantize_block(
@@ -225,9 +126,9 @@ class OptimizedRTNQuantizer(RTNQuantizer):
):
"""Apply imatrix-informed RTN quantization to a block.
- Pure-algorithm entry point. All infrastructure (device placement,
- act-max hook registration, imatrix collection, cleanup) is handled
- by the Compressor before calling this method.
+ Pure-algorithm entry point. Device placement and cleanup are handled
+ by the Compressor; act-max and imatrix hook registration are owned by
+ the quantizer hook helpers before this method is called.
Args:
block: Module already placed on the correct device(s) with act_max
diff --git a/auto_round/algorithms/quantization/sign_round/quantizer.py b/auto_round/algorithms/quantization/sign_round/quantizer.py
index e0859d260..2297d9d44 100644
--- a/auto_round/algorithms/quantization/sign_round/quantizer.py
+++ b/auto_round/algorithms/quantization/sign_round/quantizer.py
@@ -36,7 +36,6 @@ from auto_round.logger import logger
from auto_round.utils import (
check_to_quantized,
compile_func,
- convert_module_to_hp_if_necessary,
get_module,
htcore,
is_auto_device_mapping,
@@ -44,6 +43,7 @@ from auto_round.utils import (
memory_monitor,
mv_module_from_gpu,
set_amax_for_all_moe_layers,
+ set_module,
to_device,
)
from auto_round.utils.device import (
@@ -324,21 +324,39 @@ class SignRoundQuantizer(BaseQuantizers):
return best_params
def quantize_layer_outside_block(
- self, layer_name: str, input_ids: torch.Tensor, q_inputs: torch.Tensor = None, device: str = "cpu", **kwargs
+ self,
+ layer_name: str,
+ input_ids: Optional[list[torch.Tensor]] = None,
+ q_inputs: Optional[list[torch.Tensor]] = None,
+ device: str = "cpu",
+ dtype: Optional[torch.dtype] = None,
):
"""Quantize a specific layer of the model using the provided inputs.
Args:
layer_name (str): The name of the layer to quantize.
- inputs (torch.Tensor): Input data for quantization.
+ input_ids (list[torch.Tensor], optional): Input data for quantization.
q_inputs (torch.Tensor, optional): Quantized input data. Defaults to None.
device (torch.device, optional): The device to use for quantization. Defaults to torch.device("cpu").
Returns:
None
"""
+ if input_ids is None:
+ logger.info(f"using rtn to quantize {layer_name}")
+ if dtype is not None:
+ layer = get_module(self.model, layer_name)
+ set_module(self.model, layer_name, layer.to(dtype))
+ self.quantize_layer_via_rtn(
+ layer_name,
+ disable_opt_rtn=getattr(self.config, "disable_opt_rtn", True),
+ )
+ return
+
logger.info(f"quantizing layer {layer_name}")
layer = get_module(self.model, layer_name)
+ if dtype is not None:
+ layer = layer.to(dtype)
if hasattr(layer, "tuning_device"):
device = layer.tuning_device
@@ -358,7 +376,7 @@ class SignRoundQuantizer(BaseQuantizers):
static_attention_dtype,
):
tmp_inputs = q_inputs if q_inputs is not None else input_ids
- hook_handles = self._register_act_max_hook(layer)
+ hook_handles = self.register_calibration_hooks(layer)
with torch.no_grad():
for input in tmp_inputs:
layer(input)
diff --git a/auto_round/algorithms/quantization/sign_roundv2/quantizer.py b/auto_round/algorithms/quantization/sign_roundv2/quantizer.py
index 145215dd1..b8ff6c9bb 100644
--- a/auto_round/algorithms/quantization/sign_roundv2/quantizer.py
+++ b/auto_round/algorithms/quantization/sign_roundv2/quantizer.py
@@ -22,6 +22,7 @@ from torch import autocast
from auto_round.algorithms.quantization.sign_round.config import SignRoundConfig
from auto_round.algorithms.quantization.sign_round.quantizer import SignRoundQuantizer
+from auto_round.algorithms.quantization.utils import register_imatrix_hooks
from auto_round.data_type.gguf import (
double_quant_tensor_sym_rtn,
quant_tensor_gguf_asym_dq,
@@ -360,26 +361,15 @@ class SignRoundV2Quantizer(SignRoundQuantizer):
)
return super()._get_loss(output_q, current_output, indices, mse_loss, device)
- def _register_act_max_hook(self, model):
- hook_handles = super()._register_act_max_hook(model)
+ def register_calibration_hooks(self, model, *, act_max: bool = True, imatrix: bool = True):
+ hook_handles = super().register_calibration_hooks(model, act_max=act_max, imatrix=imatrix)
+ if not imatrix:
+ return hook_handles
is_wint4aint4 = ("int4" in self.act_data_type or ("int" in self.act_data_type and self.act_bits == 4)) and (
"int4" in self.data_type or ("int" in self.data_type and self.bits == 4)
)
if is_wint4aint4:
return hook_handles
-
- def get_imatrix_hook(module, input, output):
- input = input[0] if isinstance(input, (tuple, list)) else input
- flattened = input.reshape(-1, input.shape[-1]).to(torch.float32)
- squared = torch.sum(torch.pow(flattened, 2), dim=0).to(torch.float32)
-
- if not hasattr(module, "imatrix"):
- module.imatrix = squared
- else:
- module.imatrix += squared.to(module.imatrix.device)
-
- for _, module in model.named_modules():
- if isinstance(module, self.supported_types) and check_to_quantized(module):
- hook_handles.append(module.register_forward_hook(get_imatrix_hook))
+ hook_handles.extend(register_imatrix_hooks(self, model))
return hook_handles
diff --git a/auto_round/algorithms/quantization/utils.py b/auto_round/algorithms/quantization/utils.py
new file mode 100644
index 000000000..ac896183c
--- /dev/null
+++ b/auto_round/algorithms/quantization/utils.py
@@ -0,0 +1,98 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import torch
+
+from auto_round.compressors.utils import check_need_act_calibration
+from auto_round.data_type.utils import reshape_pad_tensor_by_group_size
+from auto_round.utils import SUPPORTED_LAYER_TYPES, check_to_quantized
+
+
+def register_act_max_hooks(quantizer, model):
+
+ def get_act_max_hook(module, input, output):
+ if isinstance(input, (tuple, list)):
+ input = input[0]
+ if input.numel() == 0:
+ return
+ input, _, _ = reshape_pad_tensor_by_group_size(input, quantizer.act_group_size)
+ act_max = torch.max(torch.abs(input), dim=-1).values
+ if not hasattr(module, "act_max") or module.act_max.numel() == 0:
+ module.act_max = act_max
+ if quantizer.config.is_act_nv_fp:
+ max_val = act_max.max()
+ module.act_max = max_val.unsqueeze(0) if max_val.dim() == 0 else max_val
+ else:
+ act_max = act_max.to(module.act_max.device)
+ if quantizer.config.is_act_nv_fp:
+ max_val = torch.max(act_max.max(), module.act_max.max())
+ module.act_max = max_val.unsqueeze(0) if max_val.dim() == 0 else max_val
+ else:
+ module.act_max = torch.max(act_max, module.act_max)
+
+ hook_handles = []
+ if isinstance(model, SUPPORTED_LAYER_TYPES):
+ if (
+ hasattr(model, "act_dynamic")
+ and check_need_act_calibration(model.act_dynamic, model.act_data_type, model.act_bits)
+ and check_to_quantized(model)
+ ):
+ hook_handles.append(model.register_forward_hook(get_act_max_hook))
+ return hook_handles
+
+ for name, module in model.named_modules():
+ if (
+ hasattr(module, "act_dynamic")
+ and check_need_act_calibration(module.act_dynamic, module.act_data_type, module.act_bits)
+ and check_to_quantized(module)
+ ):
+ hook_handles.append(module.register_forward_hook(get_act_max_hook))
+ continue
+
+ if name in quantizer.layer_config:
+ config = quantizer.layer_config[name]
+ act_dynamic = config.get("act_dynamic", True)
+ act_data_type = config.get("act_data_type", None)
+ act_bits = config.get("act_bits", 16)
+ if (
+ config["bits"] <= 8
+ and check_need_act_calibration(act_dynamic, act_data_type, act_bits)
+ and check_to_quantized(config)
+ ):
+ hook_handles.append(module.register_forward_hook(get_act_max_hook))
+ continue
+ return hook_handles
+
+
+def register_imatrix_hooks(quantizer, model, *, with_count: bool = False):
+
+ def get_imatrix_hook(module, input, output):
+ input = input[0] if isinstance(input, (tuple, list)) else input
+ flattened = input.reshape(-1, input.shape[-1]).to(torch.float32)
+ squared = torch.sum(torch.pow(flattened, 2), dim=0).to(torch.float32)
+
+ if not hasattr(module, "imatrix"):
+ module.imatrix = squared
+ if with_count:
+ module.imatrix_cnt = input.shape[0]
+ else:
+ module.imatrix += squared.to(module.imatrix.device)
+ if with_count:
+ module.imatrix_cnt += input.shape[0]
+
+ hook_handles = []
+ for _, module in model.named_modules():
+ if isinstance(module, quantizer.supported_types) and check_to_quantized(module):
+ hook_handles.append(module.register_forward_hook(get_imatrix_hook))
+ return hook_handles
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index 12b841ace..c2e2742c4 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -152,7 +152,13 @@ class AutoRound:
local_args = {k: v for k, v in locals().items() if k not in cls.SKIP_ARGS}
if extra_config is not None:
- local_args.update({k: v for k, v in extra_config.to_dict().items() if k in local_args and v is not None})
+ for key, value in extra_config.to_dict().items():
+ if value is None:
+ continue
+ if key in local_args:
+ local_args[key] = value
+ else:
+ kwargs[key] = value
from auto_round.compressors.entry import AutoRoundCompatible
diff --git a/auto_round/calibration/hooks.py b/auto_round/calibration/hooks.py
index ff4c6ddd9..0d356ee9f 100644
--- a/auto_round/calibration/hooks.py
+++ b/auto_round/calibration/hooks.py
@@ -105,6 +105,18 @@ def make_block_forward_func(state, name: str) -> Callable:
else: # append cache inputs
new_data = post_process_cache_data(state.quantizer.batch_size, kwargs[key], key)
if new_data is None: # shareable args or NoneType
+ if key in state.model_context.shared_cache_keys:
+ # Shared keys are normally the same across samples. However
+ # in VLM visual encoders (e.g. Qwen2-VL) ``position_embeddings``
+ # varies per image because each image has a different patch count.
+ # Upgrade from shared (raw value) to per-sample list storage so
+ # each sample gets its own positional embeddings.
+ raw_new = to_device(kwargs[key], device=torch.device("cpu"))
+ stored = state.inputs[name].get(key)
+ if isinstance(stored, list):
+ stored.append(raw_new)
+ elif stored is not None:
+ state.inputs[name][key] = [stored, raw_new]
continue
new_data = to_device(new_data, device=torch.device("cpu"))
if state.quantizer.batch_size <= 1:
diff --git a/auto_round/calibration/mllm.py b/auto_round/calibration/mllm.py
index 62dfd2448..0c403a54a 100644
--- a/auto_round/calibration/mllm.py
+++ b/auto_round/calibration/mllm.py
@@ -44,7 +44,7 @@ class MLLMCalibrator(LLMCalibrator):
from auto_round.compressors.mllm.dataset import get_mllm_dataloader
from auto_round.compressors.mllm.template import get_template
- from auto_round.special_model_handler import MISTRAL_3_2_MODELS
+ from auto_round.special_model_handler import MISTRAL_3_2_MODELS, NOT_SUPPORT_ONLY_TEXT_MODELS
c = self.compressor
mc = c.model_context
@@ -97,6 +97,13 @@ class MLLMCalibrator(LLMCalibrator):
" switching to liuhaotian/llava_conv_58k"
)
dataset = "liuhaotian/llava_conv_58k"
+ elif dataset in CALIB_DATASETS and c.template_obj.model_type in NOT_SUPPORT_ONLY_TEXT_MODELS:
+ logger.warning(
+ f"{getattr(mc.model.config, 'model_type', c.template_obj.model_type)}"
+ f" does not support for {dataset},"
+ " will use liuhaotian/llava_conv_58k with default config as an alternative."
+ )
+ dataset = "liuhaotian/llava_conv_58k"
(
c.dataloader,
c.batch_size,
@@ -114,6 +121,7 @@ class MLLMCalibrator(LLMCalibrator):
bs=bs,
seed=c.seed,
nsamples=nsamples,
+ gradient_accumulate_steps=c.gradient_accumulate_steps,
quant_nontext_module=c.quant_nontext_module,
)
else:
diff --git a/auto_round/calibration/state.py b/auto_round/calibration/state.py
index 85f28d67d..f4fdfc9f7 100644
--- a/auto_round/calibration/state.py
+++ b/auto_round/calibration/state.py
@@ -19,7 +19,7 @@ This dataclass owns every per-run calibration field shared between
- Cache state ``(inputs, to_cached_layers, last_cache_name, blocks_requiring_input_ids)``
- Per-batch shape state ``(attention_mask, batch_dim)``
-- Calibration parameters ``(batch_size, nsamples, seqlen, dataset, dataloader)``
+- Calibration parameters ``(batch_size, gradient_accumulate_steps, nsamples, seqlen, dataset, dataloader)``
Both the compressor and the quantizer hold a reference to the same
``CalibrationState`` instance (wired in ``BaseCompressor._resolve_scheme``).
@@ -61,6 +61,7 @@ class CalibrationState:
# ── Calibration parameters ─────────────────────────────────────────────
batch_size: int = 8
+ gradient_accumulate_steps: int = 1
nsamples: int = 128
seqlen: int = 2048
dataset: Any = None
@@ -87,6 +88,7 @@ class CalibrationState:
last_cache_name=getattr(compressor, "last_cache_name", None),
blocks_requiring_input_ids=getattr(compressor, "blocks_requiring_input_ids", []) or [],
batch_size=getattr(compressor, "batch_size", 8) or 8,
+ gradient_accumulate_steps=getattr(compressor, "gradient_accumulate_steps", 1) or 1,
nsamples=getattr(compressor, "nsamples", 128) or 128,
seqlen=getattr(compressor, "seqlen", 2048) or 2048,
dataset=getattr(compressor, "dataset", None),
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py
index af36c0324..c48618c4a 100644
--- a/auto_round/compressors/base.py
+++ b/auto_round/compressors/base.py
@@ -172,6 +172,7 @@ class BaseCompressor(object):
nsamples=nsamples if nsamples is not None else 128,
seqlen=seqlen if seqlen is not None else 2048,
batch_size=kwargs.pop("batch_size", 8),
+ gradient_accumulate_steps=kwargs.pop("gradient_accumulate_steps", 1),
)
# ``dataset`` is not a named __init__ parameter – it arrives via
@@ -213,6 +214,9 @@ class BaseCompressor(object):
# Extra/legacy kwargs for backward compatibility
# Major version releases may pack them with extra configuration options
+ kwargs.pop("iters", None)
+ kwargs.pop("enable_alg_ext", None)
+ kwargs.pop("vlm", None)
amp = kwargs.pop("amp", True)
nblocks = kwargs.pop("nblocks", 1)
disable_deterministic_algorithms = kwargs.pop("disable_deterministic_algorithms", True)
@@ -224,6 +228,9 @@ class BaseCompressor(object):
model_dtype = kwargs.pop("model_dtype", None)
trust_remote_code = kwargs.pop("trust_remote_code") if "trust_remote_code" in kwargs else True
quant_nontext_module = kwargs.pop("quant_nontext_module", False)
+ device = kwargs.pop("device", None)
+ if device is not None:
+ logger.warning("`device` is deprecated, please use `device_map` instead")
self.static_attention_dtype = kwargs.pop("static_attention_dtype", None)
# Attention static dtype
@@ -258,10 +265,6 @@ class BaseCompressor(object):
# instead of MATH (avoids ~10x peak-VRAM blow-up during block tuning).
patch_xpu_sdpa_drop_causal_mask()
- device = kwargs.pop("device", None)
- if device is not None:
- logger.warning("`device` is deprecated, please use `device_map` instead")
-
# Tuning hyperparameters
self.seed = seed
set_seed(self.seed)
@@ -327,7 +330,7 @@ class BaseCompressor(object):
# Apply torch compile adjustments eagerly so that ar.enable_torch_compile
# reflects the correct value immediately after construction (not only after post_init).
- self._adjust_torch_compile(enable_torch_compile)
+ self._precheck_torch_compile(enable_torch_compile)
self.compress_context.enable_torch_compile = self.enable_torch_compile
# ``self._calibration_state`` was created at the top of __init__ so
@@ -620,10 +623,8 @@ class BaseCompressor(object):
def diffusion(self):
return self.model_context.is_diffusion
- def _adjust_torch_compile(self, enable_torch_compile: bool) -> None:
- """Sets the torch compile configuration for the tuning."""
- self.enable_torch_compile = enable_torch_compile
-
+ def _get_torch_compile_guard_state(self) -> tuple[bool, bool, int]:
+ """Return raw dtype state used by torch.compile guard rules."""
# Determine fp8 / nvfp4 intent from raw config before scheme resolution.
cfg = self.quantize_config
raw_scheme = self.scheme if isinstance(self.scheme, str) else ""
@@ -641,6 +642,11 @@ class BaseCompressor(object):
)
act_bits = getattr(cfg, "act_bits", 16) or 16
+ return is_raw_fp8, is_raw_nv_fp, act_bits
+
+ def _maybe_log_torch_compile_default_hint(self) -> None:
+ """Log the default torch.compile hint once final config state is available."""
+ is_raw_fp8, _, act_bits = self._get_torch_compile_guard_state()
if (
not self.enable_torch_compile
and TORCH_VERSION_AT_LEAST_2_6
@@ -654,6 +660,13 @@ class BaseCompressor(object):
"'enable_torch_compile' is set to `False` by default. "
"Enabling it can reduce tuning cost by 20%, but it might throw an exception.",
)
+
+ def _apply_torch_compile_constraints(self, enable_torch_compile: bool) -> None:
+ """Apply torch.compile disabling rules for the current compressor state."""
+ self.enable_torch_compile = enable_torch_compile
+ cfg = self.quantize_config
+ is_raw_fp8, is_raw_nv_fp, _ = self._get_torch_compile_guard_state()
+
# On HPU, we rely on torch.compile to speed up the model execution.
if self.enable_torch_compile and is_raw_fp8 and not is_hpex_available():
self.enable_torch_compile = False
@@ -670,6 +683,22 @@ class BaseCompressor(object):
# "reset enable_torch_compile to `False` as super_group_size is set for algorithm extension"
# )
+ def _precheck_torch_compile(self, enable_torch_compile: bool) -> None:
+ """Apply early torch.compile adjustments before scheme resolution.
+
+ This runs during ``__init__`` so the compressor exposes a sensible
+ ``enable_torch_compile`` value immediately after construction, even
+ though scheme resolution has not completed yet.
+ """
+ self._apply_torch_compile_constraints(enable_torch_compile)
+
+ def _finalize_torch_compile(self) -> None:
+ """Re-evaluate torch.compile after scheme resolution with final attrs."""
+ requested_enable_torch_compile = self.enable_torch_compile
+ self._apply_torch_compile_constraints(requested_enable_torch_compile)
+ if not requested_enable_torch_compile:
+ self._maybe_log_torch_compile_default_hint()
+
def _get_calibration_dataset(self) -> str:
"""Resolve calibration dataset: self.dataset > AutoScheme.dataset > default."""
dataset = self._calibration_state.dataset
@@ -1004,8 +1033,8 @@ class BaseCompressor(object):
Preconditions:
- Phase 4 complete: ``layer_config`` is built and
``has_qlayer_outside_block`` is known.
- - ``self.quantize_config.data_type`` is the final resolved value
- (needed by :meth:`_adjust_torch_compile`).
+ - ``self.quantize_config.data_type`` is the final resolved value
+ (needed by :meth:`_finalize_torch_compile`).
Work performed:
- Applies the device map via :func:`~auto_round.utils.device.set_non_auto_device_map`.
@@ -1024,7 +1053,7 @@ class BaseCompressor(object):
"""
set_non_auto_device_map(self.model_context.model, self.compress_context.device_map)
# Re-evaluate torch.compile eligibility now that data_type is resolved.
- self._adjust_torch_compile(self.enable_torch_compile)
+ self._finalize_torch_compile()
self.compress_context.enable_torch_compile = self.enable_torch_compile
if self.compress_context.low_cpu_mem_usage:
self._offloader.reset()
@@ -1115,6 +1144,15 @@ class BaseCompressor(object):
def batch_size(self, value: int) -> None:
self._calibration_state.batch_size = value
+ @property
+ def gradient_accumulate_steps(self) -> int:
+ return self._calibration_state.gradient_accumulate_steps
+
+ @gradient_accumulate_steps.setter
+ def gradient_accumulate_steps(self, value: int) -> None:
+ if value is not None:
+ self._calibration_state.gradient_accumulate_steps = value
+
@property
def nsamples(self) -> int:
return self._calibration_state.nsamples
diff --git a/auto_round/compressors/data_driven.py b/auto_round/compressors/data_driven.py
index d3cc72e41..c78e11d6e 100644
--- a/auto_round/compressors/data_driven.py
+++ b/auto_round/compressors/data_driven.py
@@ -59,7 +59,6 @@ from auto_round.utils import (
memory_monitor,
mv_module_from_gpu,
set_amax_for_all_moe_layers,
- set_module,
to_device,
to_dtype,
wrap_block_forward_positional_to_kwargs,
@@ -68,7 +67,7 @@ from auto_round.utils.device import (
_force_trim_malloc,
parse_available_devices,
)
-from auto_round.wrapper import WrapperLinear, WrapperMultiblock
+from auto_round.wrapper import WrapperMultiblock
class DataDrivenCompressor(BaseCompressor):
@@ -300,9 +299,10 @@ class DataDrivenCompressor(BaseCompressor):
"""
from auto_round.calibration.state import CalibrationState
- assert not self.mllm and not self.diffusion, (
- f"Currently, {self.__class__.__name__} does not support quantize_block " "for MLLM / diffusion models."
- )
+ if self.diffusion:
+ raise NotImplementedError(
+ f"Currently, {self.__class__.__name__} does not support quantize_block for diffusion models."
+ )
# Ensure post_init has been called (sets up model_context, compress_context,
# quantizer, layer_config, etc.).
@@ -365,13 +365,13 @@ class DataDrivenCompressor(BaseCompressor):
# ── Infrastructure: collect reference output and act_max ──────────────
bs = self.quantizer.batch_size * self.quantizer.infer_bs_coeff
if q_input is None:
- hook_handles = self.quantizer._register_act_max_hook(block)
+ hook_handles = self.quantizer.register_calibration_hooks(block)
reference_output = self.quantizer._get_block_outputs(block, input_ids, input_others, bs)
for h in hook_handles:
h.remove()
else:
reference_output = self.quantizer._get_block_outputs(block, input_ids, input_others, bs)
- hook_handles = self.quantizer._register_act_max_hook(block)
+ hook_handles = self.quantizer.register_calibration_hooks(block)
if hook_handles:
self.quantizer._get_block_outputs(block, q_input, input_others, bs, save_output=False)
for h in hook_handles:
@@ -514,7 +514,7 @@ class DataDrivenCompressor(BaseCompressor):
# ── Infrastructure: collect reference output and act_max ──────────
bs = self.quantizer.batch_size * self.quantizer.infer_bs_coeff
if q_input is None:
- hook_handles = self.quantizer._register_act_max_hook(m)
+ hook_handles = self.quantizer.register_calibration_hooks(m)
reference_output = self.quantizer._get_block_outputs(
m, input_ids, input_others, bs, device_override=loss_device
)
@@ -524,7 +524,7 @@ class DataDrivenCompressor(BaseCompressor):
reference_output = self.quantizer._get_block_outputs(
m, input_ids, input_others, bs, device_override=loss_device
)
- hook_handles = self.quantizer._register_act_max_hook(m)
+ hook_handles = self.quantizer.register_calibration_hooks(m)
if hook_handles:
self.quantizer._get_block_outputs(
m, q_input, input_others, bs, save_output=False, device_override=loss_device
@@ -797,28 +797,12 @@ class DataDrivenCompressor(BaseCompressor):
)
layer_names.remove(layer_name)
continue
- logger.info(f"using rtn to quantize {layer_name}")
- from auto_round.data_type import QUANT_FUNC_WITH_DTYPE
-
- layer = get_module(self.model, layer_name)
- layer = layer.to(self.compress_context.device)
- layer = convert_module_to_hp_if_necessary(
- layer, self.model_context.amp_dtype, self.compress_context.device
- )
- set_module(self.model, layer_name, layer)
-
- wrapper_layer = WrapperLinear(
- layer,
- enable_round_tuning=False,
- enable_minmax_tuning=False,
- enable_norm_bias_tuning=False,
- enable_torch_compile=self.enable_torch_compile,
+ self.quantizer.quantize_layer_outside_block(
+ layer_name,
+ input_ids=None,
device=self.compress_context.device,
disable_opt_rtn=getattr(self, "disable_opt_rtn", False),
)
- new_layer = wrapper_layer.unwrapper({})
- set_module(self.model, layer_name, new_layer)
- layer.cpu()
layer_names.remove(layer_name)
if len(layer_names) == 0:
memory_monitor.update()
@@ -1020,7 +1004,7 @@ class CalibratedRTNCompressor(DataDrivenCompressor):
block = block.to(self.compress_context.device)
# ── Infrastructure: register act_max hook and run forward pass ──
- hook_handles = self.quantizer._register_act_max_hook(block)
+ hook_handles = self.quantizer.register_calibration_hooks(block, imatrix=False)
input_ids = self.quantizer._get_block_outputs(
block,
input_ids,
@@ -1077,14 +1061,14 @@ class CalibratedRTNCompressor(DataDrivenCompressor):
def _quant_rtn_with_imatrix(self) -> None:
"""Performs RTN quantization using input activation statistics (imatrix).
- This method accumulates per-channel second-moment activation statistics (imatrix)
- via forward hooks and uses them to perform RTN quantization. If CUDA memory runs out,
- it falls back to CPU-based blockwise quantization.
+ OptimizedRTNQuantizer owns imatrix hook registration. This method only
+ enables the quantizer-side collection path and keeps the OOM fallback.
Returns:
None
"""
logger.info("start to compute imatrix")
+ self.quantizer.enable_imatrix = True
# Dataloader resolution is owned by ``CalibrationState``.
self._calibration_state.ensure_dataloader(self.model_context, self.seed)
@@ -1095,30 +1079,7 @@ class CalibratedRTNCompressor(DataDrivenCompressor):
if hasattr(model, "hf_device_map") and len(model.hf_device_map) > 1:
dispatch_model(model, model.hf_device_map)
- def register_act_hook(model):
- """Registers hooks to accumulate activation squared norms into `imatrix`."""
-
- def get_imatrix_hook(module, input, output):
- input = input[0] if isinstance(input, (tuple, list)) else input
- flattened = input.reshape(-1, input.shape[-1]).to(torch.float32)
- squared = torch.sum(torch.pow(flattened, 2), dim=0).to(torch.float32)
-
- if not hasattr(module, "imatrix"):
- module.imatrix = squared
- module.imatrix_cnt = input.shape[0]
- else:
- module.imatrix += squared.to(module.imatrix.device)
- module.imatrix_cnt += input.shape[0]
-
- hook_handles = []
- for name, module in model.named_modules():
- if type(module) in SUPPORTED_LAYER_TYPES and check_to_quantized(module):
- hook = module.register_forward_hook(get_imatrix_hook)
- hook_handles.append(hook)
- return hook_handles
-
- hooks = register_act_hook(model)
-
+ hooks = self.quantizer.register_calibration_hooks(model, act_max=False)
try:
if hasattr(model, "hf_device_map") and len(model.hf_device_map) > 1:
import accelerate
@@ -1150,9 +1111,9 @@ class CalibratedRTNCompressor(DataDrivenCompressor):
except Exception as e:
raise
finally:
- # Always remove hooks
for hook in hooks:
hook.remove()
+ self.quantizer.enable_imatrix = False
def quantize(self):
"""Quantize all modules in the model using RTN (Round-To-Nearest) strategy.
diff --git a/auto_round/compressors/entry.py b/auto_round/compressors/entry.py
index 23b76cf55..e7f3fd67b 100644
--- a/auto_round/compressors/entry.py
+++ b/auto_round/compressors/entry.py
@@ -189,6 +189,8 @@ class AutoRound(object):
scheme="W4A16",
low_gpu_mem_usage: bool = False,
device_map: Union[str, torch.device, int, dict] = 0,
+ iters: int = None,
+ gradient_accumulate_steps: int = 1,
enable_torch_compile: bool = False,
seed: int = 42,
low_cpu_mem_usage: bool = True,
@@ -232,6 +234,8 @@ class AutoRound(object):
scheme=scheme,
low_gpu_mem_usage=low_gpu_mem_usage,
device_map=device_map,
+ iters=iters,
+ gradient_accumulate_steps=gradient_accumulate_steps,
enable_torch_compile=enable_torch_compile,
seed=seed,
low_cpu_mem_usage=low_cpu_mem_usage,
@@ -311,6 +315,7 @@ class AutoRound(object):
from auto_round.auto_scheme.gen_auto_scheme import AutoScheme as _AutoScheme
is_auto_scheme = isinstance(scheme, _AutoScheme)
+ quant_config.enable_imatrix = enable_imatrix
if enable_imatrix or needs_act_calib or is_auto_scheme:
quant_config._alg_cls = "OptimizedRTNQuantizer"
@@ -429,6 +434,12 @@ class AutoRoundCompatible:
from auto_round.utils import is_diffusion_model, is_mllm_model
from auto_round.utils.model import is_model_free_route
+ device = kwargs.pop("device", None)
+ if device is not None:
+ logger.warning_once("`device` is deprecated, please use `device_map` instead")
+ if device_map in (None, 0):
+ device_map = device
+
# ---- Model-free fast-path detection --------------------------------
if is_model_free_route(model, scheme, iters, kwargs.get("disable_opt_rtn"), kwargs):
from auto_round.compressors.model_free import ModelFreeCompressor
@@ -463,6 +474,12 @@ class AutoRoundCompatible:
act_sym = kwargs.pop("act_sym", None)
act_data_type = kwargs.pop("act_data_type", None)
act_dynamic = kwargs.pop("act_dynamic", None)
+ enable_opt_rtn = kwargs.pop("enable_opt_rtn", None)
+ lr = kwargs.pop("lr", None)
+ minmax_lr = kwargs.pop("minmax_lr", None)
+ enable_minmax_tuning = kwargs.pop("enable_minmax_tuning", True)
+ enable_norm_bias_tuning = kwargs.pop("enable_norm_bias_tuning", False)
+ enable_quanted_input = kwargs.pop("enable_quanted_input", True)
# Decide which algorithm to use
if iters == 0:
@@ -479,16 +496,11 @@ class AutoRoundCompatible:
act_data_type=act_data_type,
act_dynamic=act_dynamic,
disable_opt_rtn=disable_opt_rtn,
+ enable_opt_rtn=enable_opt_rtn,
**common_config_kwargs,
)
else:
# AutoRoundCompatible mode
- lr = kwargs.pop("lr", None)
- minmax_lr = kwargs.pop("minmax_lr", None)
- enable_minmax_tuning = kwargs.pop("enable_minmax_tuning", True)
- enable_norm_bias_tuning = kwargs.pop("enable_norm_bias_tuning", False)
- enable_quanted_input = kwargs.pop("enable_quanted_input", True)
-
config = SignRoundConfig(
iters=iters,
gradient_accumulate_steps=gradient_accumulate_steps,
@@ -559,6 +571,7 @@ class AutoRoundCompatible:
scheme=scheme,
dataset=dataset,
iters=iters,
+ gradient_accumulate_steps=gradient_accumulate_steps,
low_gpu_mem_usage=low_gpu_mem_usage,
device_map=device_map,
enable_torch_compile=enable_torch_compile,
diff --git a/auto_round/compressors/zero_shot.py b/auto_round/compressors/zero_shot.py
index 1cd2ced4c..3b471c6a2 100644
--- a/auto_round/compressors/zero_shot.py
+++ b/auto_round/compressors/zero_shot.py
@@ -65,7 +65,6 @@ class ZeroShotCompressor(BaseCompressor):
device_map=device_map,
low_gpu_mem_usage=low_gpu_mem_usage,
enable_torch_compile=enable_torch_compile,
- enable_alg_ext=enable_alg_ext,
seed=seed,
low_cpu_mem_usage=low_cpu_mem_usage,
**kwargs,
diff --git a/test/test_cpu/core/test_llmc_quantize_block.py b/test/test_cpu/core/test_llmc_quantize_block.py
new file mode 100644
index 000000000..28cae1345
--- /dev/null
+++ b/test/test_cpu/core/test_llmc_quantize_block.py
@@ -0,0 +1,64 @@
+from types import SimpleNamespace
+
+import torch
+
+from auto_round.compressors.data_driven import DataDrivenCompressor
+
+
+class _QuantizerStub:
+ batch_size = 1
+ infer_bs_coeff = 1
+ enable_quanted_input = False
+ act_data_type = ""
+
+ def register_calibration_hooks(self, block, *, act_max=True, imatrix=True):
+ return []
+
+ def _get_block_outputs(self, block, input_ids, input_others, bs, save_output=True):
+ return torch.ones(1, 1)
+
+ def quantize_block(
+ self, block, input_ids, input_others, reference_output, loss_device=None, mid_iter_mem_check=False
+ ):
+ return None
+
+
+def test_llmc_quantize_block_allows_mllm(monkeypatch):
+ compressor = object.__new__(DataDrivenCompressor)
+ compressor.model_context = SimpleNamespace(is_mllm=True, is_diffusion=False, amp_dtype=torch.float32)
+ compressor._post_init_done = True
+ compressor._calibration_state = SimpleNamespace(inputs={})
+ compressor.compress_context = SimpleNamespace(
+ device_map="cpu",
+ device_list=["cpu"],
+ low_gpu_mem_usage=False,
+ )
+ compressor.quantizer = _QuantizerStub()
+ compressor.quant_block_list = [["decoder.layers.0"]]
+
+ def _normalize(_inputs):
+ compressor.inputs = {"decoder.layers.0": [(torch.ones(1, 1), {})]}
+
+ def _preprocess(_block_inputs, _input_name="hidden_states"):
+ assert compressor.model_context.is_mllm is False
+ return torch.ones(1, 1), {}
+
+ compressor.normalize_decoding_layer_inputs_ = _normalize
+ compressor._preprocess_block_inputs = _preprocess
+
+ monkeypatch.setattr("auto_round.compressors.data_driven.materialize_model_", lambda block: None)
+ monkeypatch.setattr(
+ "auto_round.compressors.data_driven.convert_module_to_hp_if_necessary",
+ lambda block, amp_dtype, device: None,
+ )
+ monkeypatch.setattr("auto_round.compressors.data_driven.mv_module_from_gpu", lambda block: None)
+ monkeypatch.setattr("auto_round.compressors.data_driven.clear_memory", lambda *args, **kwargs: None)
+ monkeypatch.setattr("auto_round.compressors.data_driven.is_nv_fp", lambda *_args, **_kwargs: False)
+ monkeypatch.setattr("auto_round.compressors.data_driven.is_static_wfp8afp8", lambda *_args, **_kwargs: False)
+
+ block = torch.nn.Linear(1, 1)
+ q_outputs, reference_output = compressor.quantize_block(block, inputs=[((torch.ones(1, 1), {}),)])
+
+ assert q_outputs is None
+ assert torch.equal(reference_output, torch.ones(1, 1))
+ assert compressor.model_context.is_mllm is True